You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/05/06 06:59:01 UTC
[kylin] branch kylin5 updated (be4e981b42 -> 1d1a9d0d60)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from be4e981b42 [DIRTY] fix maven package error
new f66b85f0d2 KYLIN-5521 fix JoinsGraph
new 54940c9a9d KYLIN-5521 fix LeftOrInner Join
new 2cf578b0fa KYLIN-5522 fix local debug mode
new 520e7c67ea KYLIN-5523 computed column as join key & partition column
new 2502c15632 KYLIN-5521 fix JoinsGraph
new 2a152803bb mirror: fix complie
new f929bd876e KYLIN-5524 Supports CONCAT for variable arguments
new 09ea457278 KYLIN-5525 fix error when model init error in multi threads
new 4a889f08ab KYLIN-5526 fix unique queue async query count more than setting
new 5657afc7f2 KYLIN-5523 [FOLLOW UP] fix some function of cc as join key or filter column
new 59a2e21034 KYLIN-5527 Data count check for segment layout building
new 00e52f4b3f [DIRTY] Rewrite spark InferFiltersFromConstraints
new 842a0601a6 KYLIN-5528 support collect sparder gc
new 620a6ad5bf KYLIN-5529 Support adding "basic agg index" or "basic table index" separately
new 597a9367ec KYLIN-5530 remove repartition write KYLIN-5530 Optimized snapshot builds KYLIN-5530 Flat Table Repartition before writing data source tables/directories
new 80a87f9d53 [DIRTY] Fix a build job is submitted in yarn cluster mode, it cannot be written into the driver log
new 2e60700e52 KYLIN-5521 [FOLLOW UP] fix LeftOrInner Join & unstable UT
new e39b34cb46 KYLIN-5531 remove redundancy code
new 2e83b6e8a5 mirror: fix ut & compile
new 96d806eae8 KYLIN-5532 When calcite optimize, can cancel query search
new 2e29f00053 KYLIN-5533 add model list lite api
new df6ef0a5ae KYLIN-5534 Fix the inconsistent behavior of exporting tds between normal users and admin
new 625c4b6c7e KYLIN-5521 [FOLLOW UP] Fix JoinsGraph toString method StackOverFlow
new 9bf63e0183 KYLIN-5535 Fix the username suffix matching
new 5d251e3139 KYLIN-5536 Limit the segment range of MAX query to improve query performance
new c8a464d2ca KYLIN-5533 [FOLLOW UP] fix BeanUtils copyProperties NPE
new 23bf641c01 KYLIN-5523 [FOLLOW UP] Fix the error caused by using ceil/floor functions in model filtering conditions.
new 10a180422c mirror: fix snyk, upgrade tomcat-embed-core from 9.0.69 to 9.0.71 fix snyk, upgrade commons-fileupload from 1.3.3 to 1.5
new 9af54c0484 KYLIN-5534 [FOLLOW UP] Set 'kylin.model.skip-check-tds' to true
new fb90074de2 KYLIN-5523 [FOLLOW UP] Fix using a computed column as the join key leads to a schema change failure
new 6309697701 KYLIN-5523 [FOLLOW UP] fix generate flat table sql
new cb3f4e604f KYLIN-5523 [FOLLOW UP] expand the filter condition expression to ensure compatibility
new b10a87b65d KYLIN-5530 [FOLLOW UP] fix config kylin.engine.persist-flat-use-snapshot-enabled
new e0599cbaed [DIRTY] release kyspark 4.6.6.0
new 111723d7a4 KYLIN-5537 Upgrade version, resolve vulnerability
new e19d2918c5 mirror: fix UT for snapshot building
new f508aa3439 mirror: remove unused import
new 1d1a9d0d60 [DIRTY] Fix Sonar
The 38 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pom.xml | 12 +-
.../src/main/resources/config/init.properties | 2 +-
.../kylin/rest/controller/NBasicController.java | 2 +-
.../apache/kylin/rest/KylinPrepareEnvListener.java | 2 +-
.../kylin/rest/aspect/InsensitiveNameAspect.java | 2 +-
.../apache/kylin/rest/broadcaster/Broadcaster.java | 2 +-
.../rest/config/initialize/AclTCRListener.java | 2 +-
.../rest/config/initialize/CacheCleanListener.java | 2 +-
.../rest/config/initialize/MetricsRegistry.java | 2 +-
.../initialize/TableSchemaChangeListener.java | 2 +-
.../ResourceGroupEntityValidator.java | 2 +-
.../ResourceGroupKylinInstanceValidator.java | 2 +-
.../ResourceGroupMappingInfoValidator.java | 2 +-
.../interceptor/RepeatableRequestBodyFilter.java | 2 +-
.../interceptor/ResourceGroupCheckerFilter.java | 2 +-
.../apache/kylin/rest/monitor/MonitorReporter.java | 2 +-
.../org/apache/kylin/rest/request/UserRequest.java | 2 +-
.../apache/kylin/rest/service/AccessService.java | 2 +-
.../kylin/rest/service/AsyncTaskService.java | 2 +-
.../apache/kylin/rest/service/BasicService.java | 2 +-
.../apache/kylin/rest/service/LdapUserService.java | 2 +-
.../rest/service/MockQuerySmartSupporter.java} | 18 +-
.../kylin/rest/service/OpenUserGroupService.java | 2 +-
.../apache/kylin/rest/service/ProjectService.java | 2 +-
.../apache/kylin/rest/service/SystemService.java | 7 +-
.../apache/kylin/rest/service/UserAclService.java | 2 +-
.../java/org/apache/kylin/tool/util/ToolUtil.java | 2 +-
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../java/org/apache/kylin/common/KapConfig.java | 8 +
.../java/org/apache/kylin/common/KylinConfig.java | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 59 +-
.../java/org/apache/kylin/common/KylinVersion.java | 2 +-
.../java/org/apache/kylin/common/StorageURL.java | 2 +-
.../apache/kylin/common/debug/BackdoorToggles.java | 2 +-
.../org/apache/kylin/common/msg/CnMessage.java | 2 +-
.../java/org/apache/kylin/common/msg/Message.java | 2 +-
.../kylin/common/persistence/StringEntity.java | 2 +-
.../persistence/metadata/JdbcMetadataStore.java | 2 +-
.../transaction/AuditLogReplayWorker.java | 2 +-
.../kylin/common/util/DaemonThreadFactory.java | 2 +-
.../org/apache/kylin/common/util/DateFormat.java | 2 +-
.../org/apache/kylin/common/util/EncryptUtil.java | 2 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 2 +-
.../org/apache/kylin/common/util/JsonUtil.java | 1 +
.../kylin/common/util/OrderedProperties.java | 2 +-
.../org/apache/kylin/common/util/SSHClient.java | 2 +-
.../util/{StringUtil.java => StringHelper.java} | 104 +++-
.../java/org/apache/kylin/common/util/Unsafe.java | 2 +-
.../org/apache/kylin/common/util/ZipFileUtils.java | 2 +-
.../org/apache/kylin/rest/util/PagingUtil.java | 2 +-
.../kylin-backward-compatibility.properties | 1 +
.../src/main/resources/kylin-defaults0.properties | 2 +-
.../apache/kylin/common/KylinConfigBaseTest.java | 18 +
.../common/util/NLocalFileMetadataTestCase.java | 4 +-
.../apache/kylin/common/util/StringHelperTest.java | 104 ++++
.../apache/kylin/engine/spark/ExecutableUtils.java | 2 +-
.../apache/kylin/job/constant/JobStatusEnum.java | 6 +
.../org/apache/kylin/job/dao/ExecutablePO.java | 2 +-
.../apache/kylin/job/engine/JobEngineConfig.java | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 28 +-
.../kylin/job/execution/DefaultExecutable.java | 45 +-
.../job/execution/DefaultExecutableOnModel.java | 10 +-
.../job/execution/EmailNotificationContent.java | 18 +-
.../kylin/job/execution/ExecutableParams.java | 2 +-
.../kylin/job/execution/ExecutableState.java | 11 +-
.../apache/kylin/job/execution/ExecuteResult.java | 23 +-
.../kylin/job/execution/NExecutableManager.java | 12 +
.../org/apache/kylin/job/JoinedFlatTableTest.java | 242 --------
.../job/execution/NExecutableManagerTest.java | 23 +-
.../job/impl/threadpool/NDefaultSchedulerTest.java | 2 +-
.../apache/kylin/dimension/DimensionEncoding.java | 6 +-
.../apache/kylin/metadata/acl/AclTCRManager.java | 2 +-
.../metadata/cube/cuboid/AggIndexMatcher.java | 2 +-
.../kylin/metadata/cube/cuboid/IndexMatcher.java | 2 +-
.../metadata/cube/cuboid/KECuboidSchedulerV1.java | 2 +-
.../kylin/metadata/cube/model/IndexPlan.java | 28 +-
.../kylin/metadata/cube/model/NBatchConstants.java | 1 +
.../kylin/metadata/cube/model/NDataLayout.java | 19 +-
.../cube/model/NDataLoadingRangeManager.java | 2 +-
.../kylin/metadata/cube/model/NDataSegDetails.java | 33 +-
.../kylin/metadata/cube/model/NDataSegment.java | 65 +-
.../cube/model/NDataflowCapabilityChecker.java | 2 +-
.../metadata/cube/model/NDataflowManager.java | 4 +-
.../metadata/cube/model/NIndexPlanManager.java | 2 +-
.../metadata/cube/model/SegmentPartition.java | 2 +-
.../cube/storage/ProjectStorageInfoCollector.java | 2 +-
.../kylin/metadata/datatype/BooleanSerializer.java | 2 +-
.../apache/kylin/metadata/datatype/DataType.java | 2 +-
.../apache/kylin/metadata/epoch/EpochManager.java | 2 +-
.../metadata/filter/function/LikeMatchers.java | 2 +-
.../insensitive/ModelInsensitiveRequest.java | 2 +-
.../insensitive/ProjectInsensitiveRequest.java | 2 +-
.../insensitive/UserInsensitiveRequest.java | 2 +-
.../kylin/metadata/model/ColExcludedChecker.java | 4 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 109 ++--
.../kylin/metadata/model/ComputedColumnDesc.java | 2 +-
.../org/apache/kylin/metadata/model/ISegment.java | 2 +
.../kylin/metadata/model/JoinedFlatTable.java | 371 -----------
.../apache/kylin/metadata/model/JoinsGraph.java | 607 ------------------
.../apache/kylin/metadata/model/MeasureDesc.java | 2 +-
.../kylin/metadata/model/ModelDimensionDesc.java | 4 +-
.../metadata/model/ModelJoinRelationTypeEnum.java | 2 +-
.../kylin/metadata/model/MultiPartitionDesc.java | 2 +-
.../apache/kylin/metadata/model/NDataModel.java | 30 +-
.../kylin/metadata/model/NDataModelManager.java | 2 +-
.../metadata/model/NTableMetadataManager.java | 2 +-
.../apache/kylin/metadata/model/PartitionDesc.java | 10 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 34 +-
.../metadata/model/alias/ExpressionComparator.java | 2 +-
.../model/graph/DefaultJoinEdgeMatcher.java | 69 +++
.../apache/kylin/metadata/model/graph/Edge.java | 148 +++++
.../metadata/model/graph/IJoinEdgeMatcher.java} | 14 +-
.../kylin/metadata/model/graph/JoinsGraph.java | 515 ++++++++++++++++
.../model/schema/AffectedModelContext.java | 2 +-
.../metadata/model/schema/ModelEdgeCollector.java | 2 +-
.../kylin/metadata/model/tool/CalciteParser.java | 12 +-
.../metadata/model/util/ComputedColumnUtil.java | 78 +--
.../kylin/metadata/project/ProjectInstance.java | 6 +-
.../metadata/query/JdbcQueryHistoryStore.java | 2 +-
.../realization/RealizationRuntimeException.java} | 15 +-
.../kylin/metadata/realization/SQLDigest.java | 2 +-
.../recommendation/candidate/RawRecItemTable.java | 2 +-
.../recommendation/entity/CCRecItemV2.java | 2 +-
.../metadata/recommendation/ref/OptRecV2.java | 2 +-
.../metadata/sourceusage/SourceUsageManager.java | 2 +-
.../kylin/metadata/streaming/KafkaConfig.java | 2 +-
.../kylin/metadata/user/NKylinUserManager.java | 20 +-
.../kylin/rest/security/ExternalAclProvider.java | 2 +-
.../apache/kylin/rest/security/UserAclManager.java | 2 +-
.../kylin/source/adhocquery/IPushDownRunner.java | 18 +-
.../cube/planner/algorithm/AlgorithmTestBase.java | 6 +-
.../metadata/epoch/EpochUpdateLockManagerTest.java | 2 +-
.../kylin/metadata/model/JoinsGraphTest.java | 106 +++-
.../kylin/metadata/model/MockJoinGraphBuilder.java | 1 +
.../model/util/ComputedColumnUtilTest.java | 4 +-
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metadata/user/NKylinUserManagerTest.java | 12 +
.../kylin/metrics/HdfsCapacityMetricsTest.java | 4 +-
.../apache/kylin/common/metrics/MetricsGroup.java | 2 +-
.../common/metrics/MetricsInfluxdbReporter.java | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../kylin/rest/controller/BaseController.java | 2 +-
.../kylin/rest/controller/JobController.java | 2 +-
.../kylin/rest/controller/SampleController.java | 6 +-
.../kylin/rest/controller/SegmentController.java | 2 +-
.../rest/controller/open/OpenSampleController.java | 7 +-
.../open/OpenStreamingJobController.java | 2 +-
.../rest/controller/v2/SegmentControllerV2.java | 2 +-
.../rest/controller/SampleControllerTest.java | 2 +-
.../controller/StreamingJobControllerTest.java | 2 +-
.../controller/open/OpenSampleControllerTest.java | 12 +-
.../open/OpenStreamingJobControllerTest.java | 2 +-
.../kylin/rest/response/ExecutableResponse.java | 8 +-
.../rest/response/ExecutableStepResponse.java | 3 +
.../org/apache/kylin/rest/service/JobService.java | 14 +-
.../kylin/rest/service/ModelBuildService.java | 2 +-
.../apache/kylin/rest/service/SnapshotService.java | 101 ++-
.../kylin/rest/service/TableSamplingService.java | 2 +-
.../apache/kylin/rest/service/JobServiceTest.java | 20 +-
.../kylin/rest/service/ModelServiceBuildTest.java | 16 +-
.../org/apache/kylin/rest/service/StageTest.java | 3 +
.../sdk/datasource/PushDownRunnerSDKImpl.java | 4 +-
.../sdk/datasource/adaptor/DefaultAdaptor.java | 2 +-
.../sdk/datasource/framework/JdbcConnector.java | 2 +-
.../datasource/framework/conv/ParamNodeParser.java | 2 +-
.../datasource/framework/conv/SqlConverter.java | 2 +-
.../apache/kylin/rest/service/SparkDDLService.java | 2 +-
.../kylin/rest/service/SparkSourceService.java | 22 +-
.../apache/kylin/rest/service/TableService.java | 55 +-
.../metadata/_global/project/index_build_test.json | 40 ++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 70 +++
.../b2f206e1-7a15-c94a-20f5-f608d550ead6.json | 69 +++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 204 ++++++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 241 ++++++++
.../index_build_test/table/SSB.CUSTOMER.json | 77 +++
.../index_build_test/table/SSB.LINEORDER.json | 131 ++++
.../rest/config/initialize/JobSyncListener.java | 2 +-
.../rest/config/initialize/JobSchedulerTest.java | 2 +-
.../org/apache/kylin/event/SchemaChangeTest.java | 5 +-
.../kylin/metadata/model/AntiFlatCheckerTest.java | 4 +-
.../metadata/model/ColExcludedCheckerTest.java | 4 +-
.../kylin/newten/NBadQueryAndPushDownTest.java | 6 +-
.../org/apache/kylin/newten/TableIndexTest.java | 19 +
.../java/org/apache/kylin/query/KylinTestBase.java | 16 +-
.../java/org/apache/kylin/query/NKapQueryTest.java | 2 +-
.../rest/controller/NProjectControllerTest.java | 2 +-
.../java/org/apache/kylin/util/ExecAndComp.java | 3 +-
.../rest/controller/open/OpenModelController.java | 5 +-
.../rest/controller/NIndexPlanController.java | 7 +-
.../kylin/rest/controller/NModelController.java | 13 +-
.../kylin/rest/controller/NProjectController.java | 2 +-
.../kylin/rest/controller/NTableController.java | 24 +-
.../kylin/rest/controller/NUserController.java | 2 +-
.../rest/controller/NUserGroupController.java | 2 +-
.../rest/controller/v2/NAccessControllerV2.java | 2 +-
.../controller/open/OpenModelControllerTest.java | 4 +-
.../rest/controller/NModelControllerTest.java | 70 ++-
.../rest/controller/NTableControllerTest.java | 12 +-
.../kylin/rest/controller/NUserControllerTest.java | 2 +-
.../apache/kylin/rest/model/FuzzyKeySearcher.java | 2 +-
.../apache/kylin/rest/request/ModelRequest.java | 6 +
.../apache/kylin/rest/response/IndexResponse.java | 4 +
.../rest/response/NDataModelLiteResponse.java | 129 ++++
.../kylin/rest/response/NDataModelResponse.java | 18 +
.../kylin/rest/response/NDataSegmentResponse.java | 4 +-
.../response/SynchronizedCommentsResponse.java | 2 +-
.../kylin/rest/service/AbstractModelService.java | 2 +-
.../kylin/rest/service/BaseIndexUpdateHelper.java | 34 +-
.../kylin/rest/service/FusionIndexService.java | 24 +-
.../kylin/rest/service/FusionModelService.java | 2 +-
.../kylin/rest/service/IndexPlanService.java | 52 +-
.../kylin/rest/service/ModelSemanticHelper.java | 43 +-
.../apache/kylin/rest/service/ModelService.java | 206 ++++---
.../apache/kylin/rest/service/ModelTdsService.java | 22 +-
...{ModelQueryParams.java => IndexPlanParams.java} | 36 +-
.../rest/service/params/ModelQueryParams.java | 2 +
.../rest/service/params/PaginationParams.java} | 22 +-
.../org/apache/kylin/rest/util/ModelUtils.java | 2 +-
.../org/apache/kylin/tool/bisync/BISyncTool.java | 4 +-
.../apache/kylin/tool/bisync/SyncModelBuilder.java | 5 -
.../rest/response/NDataModelResponseTest.java | 106 ++++
.../apache/kylin/rest/service/BaseIndexTest.java | 49 +-
.../kylin/rest/service/FusionModelServiceTest.java | 2 +-
.../service/ModelServiceSemanticUpdateTest.java | 55 +-
.../kylin/rest/service/ModelServiceTest.java | 122 ++--
.../service/ModelTdsServiceColumnNameTest.java | 1 +
.../kylin/rest/service/ModelTdsServiceTest.java | 69 ++-
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../kylin/rest/service/TableServiceTest.java | 134 ++--
.../kylin/tool/bisync/SyncModelBuilderTest.java | 2 +-
.../tool/bisync/tableau/TableauDatasourceTest.java | 5 +-
.../src/main/resources/config/init.properties | 2 +-
.../org/apache/kylin/query/util/EscapeParser.jj | 42 +-
.../org/apache/kylin/query/SlowQueryDetector.java | 9 +-
.../apache/kylin/query/relnode/OLAPContext.java | 4 +-
.../kylin/query/routing/RealizationChooser.java | 7 +-
.../kylin/query/routing/RealizationPruner.java | 177 ++++--
.../org/apache/kylin/query/schema/OLAPTable.java | 6 +-
.../query/security/AccessDeniedException.java | 2 +-
.../org/apache/kylin/query/security/RowFilter.java | 2 +-
.../apache/kylin/query/util/AsyncQueryUtil.java | 2 +-
.../org/apache/kylin/query/util/EscapeDialect.java | 4 +
.../apache/kylin/query/util/EscapeFunction.java | 16 +
.../query/util/ModelViewSqlNodeComparator.java | 2 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 412 +++++++++----
.../apache/kylin/query/util/QueryAliasMatcher.java | 55 +-
.../org/apache/kylin/query/util/QueryUtil.java | 431 +++----------
.../rest/controller/NAsyncQueryController.java | 10 +-
.../kylin/rest/controller/NQueryController.java | 4 +-
.../controller/SparkMetricsControllerTest.java | 2 +-
.../apache/kylin/rest/response/SQLResponseV2.java | 2 +-
.../kylin/rest/service/ModelQueryService.java | 2 +-
.../apache/kylin/rest/service/MonitorService.java | 2 +-
.../kylin/rest/service/QueryHistoryService.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 16 +-
.../kylin/rest/util/AsyncQueryRequestLimits.java | 8 +-
.../kylin/rest/util/QueryCacheSignatureUtil.java | 2 +-
.../org/apache/kylin/rest/util/QueryUtils.java | 2 +-
.../kylin/rest/service/ModelQueryServiceTest.java | 18 +-
.../kylin/rest/service/ModelServiceQueryTest.java | 103 ++--
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/rest/service/QueryServiceTest.java | 40 +-
src/query/pom.xml | 9 +-
.../optrule/AbstractAggCaseWhenFunctionRule.java | 18 +-
.../optrule/CountDistinctCaseWhenFunctionRule.java | 17 +-
.../query/optrule/KapAggFilterTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggJoinTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggProjectMergeRule.java | 4 +-
.../query/optrule/KapAggProjectTransposeRule.java | 4 +-
.../query/optrule/KapCountDistinctJoinRule.java | 4 +-
.../kap/query/optrule/KapSumCastTransposeRule.java | 19 +-
.../main/java/org/apache/kylin/query/QueryCli.java | 78 ---
.../kylin/query/engine/AsyncQueryApplication.java | 2 +-
.../apache/kylin/query/engine/AsyncQueryJob.java | 2 +-
.../kylin/query/engine/QueryRoutingEngine.java | 16 +-
.../kylin/query/udf/stringUdf/ConcatUDF.java | 45 +-
.../org/apache/kylin/query/util/RuleUtils.java | 150 +++++
.../kylin/query/engine/AsyncQueryJobTest.java | 2 +-
.../kylin/query/engine/SelectRealizationTest.java | 7 +-
.../kylin/query/engine/view/ModelViewTest.java | 2 +-
.../kylin/query/schema/KylinSqlValidatorTest.java | 2 +-
.../org/apache/kylin/query/udf/StringUDFTest.java | 21 +-
.../apache/kylin/query/util/CCOnRealModelTest.java | 11 +-
.../query/util/EscapeTransformerCalciteTest.java | 22 +
.../query/util/EscapeTransformerSparkSqlTest.java | 11 +
.../apache/kylin/query/util/PushDownUtilTest.java | 251 +++++++-
.../org/apache/kylin/query/util/QueryUtilTest.java | 344 +++++------
.../kap/secondstorage/SecondStorageLockTest.java | 6 +-
.../kap/secondstorage/tdvt/TDVTHiveTest.java | 4 +-
.../kap/clickhouse/ClickHouseStorage.java | 2 +-
.../kyligence/kap/clickhouse/job/ClickHouse.java | 15 +-
.../kap/clickhouse/job/ClickHouseLoad.java | 2 +-
.../kap/clickhouse/job/ClickhouseLoadFileLoad.java | 2 +-
.../kap/clickhouse/job/HadoopMockUtil.java | 2 +-
.../management/OpenSecondStorageEndpoint.java | 2 +-
.../management/SecondStorageEndpoint.java | 2 +-
.../service/ModelServiceWithSecondStorageTest.java | 4 +-
.../io/kyligence/kap/secondstorage/ddl/DDL.java | 2 +-
.../kap/secondstorage/ddl/exp/TableIdentifier.java | 2 +-
.../kap/secondstorage/metadata/Manager.java | 2 +-
.../org/apache/kylin/rest/QueryNodeFilter.java | 2 +-
.../apache/kylin/rest/ZookeeperClusterManager.java | 2 +-
.../org/apache/kylin/rest/config/CorsConfig.java | 2 +-
.../rest/discovery/KylinServiceDiscoveryCache.java | 4 +-
.../discovery/KylinServiceDiscoveryClient.java | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../source/hive/BeelineOptionsProcessorTest.java | 2 +-
src/spark-project/engine-spark/pom.xml | 5 +
.../engine/spark/application/SparkApplication.java | 46 +-
.../builder/PartitionDictionaryBuilderHelper.java | 2 +-
.../spark/job/DefaultSparkBuildJobHandler.java | 2 +-
.../spark/job/ExecutableAddCuboidHandler.java | 48 +-
.../kylin/engine/spark/job/NSparkCubingStep.java | 42 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 6 +-
.../job/SparkCleanupTransactionalTableStep.java | 2 +-
.../spark/merger/AfterBuildResourceMerger.java | 4 +-
.../merger/AfterMergeOrRefreshResourceMerger.java | 4 +-
.../kylin/engine/spark/mockup/CsvSource.java | 2 +-
.../kylin/engine/spark/mockup/CsvTableReader.java | 2 +-
.../engine/spark/source/NSparkDataSource.java | 2 +-
.../spark/source/NSparkMetadataExplorer.java | 2 +-
.../spark/stats/analyzer/TableAnalyzerJob.java | 2 +-
.../spark/stats/utils/DateTimeCheckUtils.java | 2 +-
.../engine/spark/utils/ComputedColumnEvalUtil.java | 52 +-
.../spark/utils/HiveTransactionTableHelper.java | 2 +-
.../engine/spark/utils/SparkJobFactoryUtils.java | 2 +-
.../engine/spark/builder/CreateFlatTable.scala | 135 +---
.../engine/spark/builder/DFBuilderHelper.scala | 12 +-
.../spark/builder/DictionaryBuilderHelper.java | 2 +-
.../engine/spark/builder/PartitionFlatTable.scala | 69 ---
.../engine/spark/builder/SegmentFlatTable.scala | 683 ---------------------
.../engine/spark/builder/SnapshotBuilder.scala | 11 +-
.../kylin/engine/spark/job/BuildJobInfos.scala | 21 +-
.../apache/kylin/engine/spark/job/DFMergeJob.java | 2 +-
.../kylin/engine/spark/job/FlatTableHelper.scala | 16 +-
.../kylin/engine/spark/job/PartitionExec.scala | 4 +
.../engine/spark/job/RDPartitionBuildExec.scala | 35 +-
.../engine/spark/job/RDSegmentBuildExec.scala | 32 +-
.../kylin/engine/spark/job/RDSegmentBuildJob.java | 11 +-
.../kylin/engine/spark/job/SegmentBuildJob.java | 16 +-
.../kylin/engine/spark/job/SegmentExec.scala | 43 +-
.../apache/kylin/engine/spark/job/SegmentJob.java | 4 +
.../kylin/engine/spark/job/SegmentMergeJob.java | 5 +-
.../kylin/engine/spark/job/SnapshotBuildJob.java | 2 +-
.../kylin/engine/spark/job/stage/StageExec.scala | 113 ++--
.../engine/spark/job/stage/build/BuildLayer.scala | 23 +-
.../engine/spark/job/stage/build/BuildStage.scala | 11 +-
.../job/stage/build/FlatTableAndDictBase.scala | 139 +++--
.../job/stage/build/GatherFlatTableStats.scala | 2 -
.../spark/job/stage/build/GenerateFlatTable.scala | 145 ++++-
.../stage/build/MaterializedFactTableView.scala | 24 +-
.../build/partition/PartitionBuildLayer.scala | 23 +-
.../build/partition/PartitionBuildStage.scala | 6 -
.../partition/PartitionFlatTableAndDictBase.scala | 22 +
.../PartitionMaterializedFactTableView.scala | 30 +-
.../engine/spark/job/stage/merge/MergeStage.scala | 2 +-
.../merge/partition/PartitionMergeStage.scala | 2 +-
.../spark/smarter/IndexDependencyParser.scala | 55 +-
.../engine/spark/job/NSparkExecutableTest.java | 15 +-
.../GenerateFlatTableWithSparkSessionTest.java | 444 ++++++++++++++
.../spark/builder/TestDimensionTableStat.scala | 13 +-
.../engine/spark/builder/TestFlatTable.scala} | 41 +-
.../engine/spark/builder/TestInferFilters.scala | 18 +-
.../spark/builder/TestSegmentFlatTable.scala | 35 +-
.../engine/spark/job/TestRDSegmentBuildExec.scala | 111 ++++
.../PartitionFlatTableAndDictBaseTest.scala | 88 +++
.../org/apache/kylin/source/jdbc/JdbcSource.java | 2 +-
.../query/pushdown/PushDownRunnerJdbcImpl.java | 4 +-
.../query/pushdown/PushDownRunnerSparkImpl.java | 3 +-
.../asyncprofiler/AsyncProfiling.scala | 2 +-
.../QueryAsyncProfilerDriverPlugin.scala | 2 +-
.../QueryAsyncProfilerSparkPlugin.scala | 2 +-
.../query/plugin/diagnose/DiagnoseConstant.scala} | 25 +-
.../diagnose/DiagnoseDriverPlugin.scala} | 50 +-
.../plugin/diagnose/DiagnoseExecutorPlugin.scala | 124 ++++
.../query/plugin/diagnose/DiagnoseHelper.scala | 62 ++
.../diagnose/DiagnoseSparkPlugin.scala} | 9 +-
.../kylin/query/runtime/ExpressionConverter.scala | 2 +-
.../scala/org/apache/spark/sql/KylinSession.scala | 56 +-
.../scala/org/apache/spark/sql/SparderEnv.scala | 36 +-
.../pushdown/PushDownRunnerSparkImplTest.java | 15 +-
.../SparkPluginWithMeta.scala} | 4 +-
.../asyncprofiler/AsyncProfilingTest.scala | 5 +-
.../QueryAsyncProfilerSparkPluginTest.scala | 2 +-
.../QueryProfilerDriverPluginTest.scala} | 5 +-
.../diagnose/DiagnoseExecutorPluginTest.scala | 109 ++++
.../query/plugin/diagnose/DiagnosePluginTest.scala | 82 +++
.../kylin/engine/spark/utils/Repartitioner.java | 15 +-
.../kylin/engine/spark/job/NSparkCubingUtil.java | 13 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java | 2 +-
.../datasource/storage/LayoutFormatWriter.scala | 98 +++
.../sql/datasource/storage/StorageStore.scala | 51 +-
.../RewriteInferFiltersFromConstraints.scala | 101 +++
.../asyncprofiler/AsyncProfilerToolTest.java | 2 +-
.../RewriteInferFiltersFromConstraintsSuite.scala | 320 ++++++++++
.../scala/io/kyligence/kap/common/SSSource.scala | 89 +++
.../scala/org/apache/kylin/common/SSSource.scala | 5 +-
.../kylin/it/TestQueryAndBuildFunSuite.scala | 1 +
.../kylin/rest/service/StreamingJobService.java | 2 +-
.../org/apache/kylin/kafka/util/KafkaUtils.java | 2 +-
.../streaming/jobs/AbstractSparkJobLauncher.java | 2 +-
.../streaming/manager/StreamingJobManager.java | 2 +-
.../apache/kylin/streaming/rest/RestSupport.java | 2 +-
.../kylin/streaming/CreateStreamingFlatTable.scala | 24 +-
.../kylin/tool/AbstractInfoExtractorTool.java | 2 +-
.../java/org/apache/kylin/tool/AuditLogTool.java | 2 +-
.../org/apache/kylin/tool/ClickhouseDiagTool.java | 4 +-
.../main/java/org/apache/kylin/tool/ConfTool.java | 2 +-
.../apache/kylin/tool/DumpHadoopSystemProps.java | 2 +-
.../org/apache/kylin/tool/JobDiagInfoTool.java | 2 +-
.../java/org/apache/kylin/tool/KylinLogTool.java | 2 +-
.../org/apache/kylin/tool/daemon/KapGuardian.java | 2 +-
.../kylin/tool/garbage/DataflowCleanerCLI.java | 2 +-
.../apache/kylin/tool/obf/KylinConfObfuscator.java | 2 +-
.../kylin/tool/upgrade/CheckProjectModeCLI.java | 2 +-
.../kylin/tool/upgrade/DeleteFavoriteQueryCLI.java | 2 +-
.../apache/kylin/tool/upgrade/RenameEntity.java | 2 +-
.../kylin/tool/upgrade/RenameUserResourceTool.java | 2 +-
.../apache/kylin/tool/upgrade/UpdateModelCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateProjectCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateSessionTableCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateUserGroupCLI.java | 2 +-
.../kylin/tool/util/HadoopConfExtractor.java | 2 +-
.../org/apache/kylin/tool/util/ServerInfoUtil.java | 2 +-
.../org/apache/kylin/tool/StorageCleanerTest.java | 2 +-
.../kylin/tool/garbage/SnapshotCleanerTest.java | 2 +-
.../apache/kylin/tool/general/CryptToolTest.java | 2 +-
.../tool/upgrade/RenameUserResourceToolTest.java | 2 +-
429 files changed, 7592 insertions(+), 4788 deletions(-)
copy src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java => common-service/src/main/java/org/apache/kylin/rest/service/MockQuerySmartSupporter.java} (68%)
rename src/core-common/src/main/java/org/apache/kylin/common/util/{StringUtil.java => StringHelper.java} (63%)
create mode 100644 src/core-common/src/test/java/org/apache/kylin/common/util/StringHelperTest.java
delete mode 100644 src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/DefaultJoinEdgeMatcher.java
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/Edge.java
copy src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java => core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/IJoinEdgeMatcher.java} (70%)
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
copy src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java => core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java} (70%)
create mode 100644 src/examples/test_case_data/localmeta/metadata/_global/project/index_build_test.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow/3ec47efc-573a-9304-4405-8e05ae184322.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow_details/3ec47efc-573a-9304-4405-8e05ae184322/b2f206e1-7a15-c94a-20f5-f608d550ead6.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/index_build_test/index_plan/3ec47efc-573a-9304-4405-8e05ae184322.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/index_build_test/model_desc/3ec47efc-573a-9304-4405-8e05ae184322.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.CUSTOMER.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.LINEORDER.json
create mode 100644 src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java
copy src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/{ModelQueryParams.java => IndexPlanParams.java} (62%)
copy src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java => modeling-service/src/main/java/org/apache/kylin/rest/service/params/PaginationParams.java} (69%)
delete mode 100644 src/query/src/main/java/org/apache/kylin/query/QueryCli.java
create mode 100644 src/query/src/main/java/org/apache/kylin/query/util/RuleUtils.java
delete mode 100644 src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/PartitionFlatTable.scala
delete mode 100644 src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
create mode 100644 src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTableWithSparkSessionTest.java
copy src/spark-project/engine-spark/src/{main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala => test/scala/org/apache/kylin/engine/spark/builder/TestFlatTable.scala} (57%)
create mode 100644 src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/TestRDSegmentBuildExec.scala
create mode 100644 src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBaseTest.scala
rename src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{ => plugin}/asyncprofiler/AsyncProfiling.scala (98%)
copy src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{ => plugin}/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala (97%)
copy src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{ => plugin}/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala (95%)
copy src/{query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java => spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseConstant.scala} (69%)
rename src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{asyncprofiler/QueryAsyncProfilerDriverPlugin.scala => plugin/diagnose/DiagnoseDriverPlugin.scala} (51%)
create mode 100644 src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPlugin.scala
create mode 100644 src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseHelper.scala
rename src/spark-project/sparder/src/main/scala/org/apache/kylin/query/{asyncprofiler/QueryAsyncProfilerSparkPlugin.scala => plugin/diagnose/DiagnoseSparkPlugin.scala} (72%)
rename src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{asyncprofiler/AsyncPluginWithMeta.scala => plugin/SparkPluginWithMeta.scala} (94%)
rename src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{ => plugin}/asyncprofiler/AsyncProfilingTest.scala (94%)
rename src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{ => plugin}/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala (96%)
rename src/spark-project/sparder/src/test/scala/org/apache/kylin/query/{asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala => plugin/asyncprofiler/QueryProfilerDriverPluginTest.scala} (92%)
create mode 100644 src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPluginTest.scala
create mode 100644 src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnosePluginTest.scala
create mode 100644 src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/LayoutFormatWriter.scala
create mode 100644 src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraints.scala
create mode 100644 src/spark-project/spark-common/src/test/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraintsSuite.scala
create mode 100644 src/spark-project/spark-it/src/test/scala/io/kyligence/kap/common/SSSource.scala
[kylin] 09/38: KYLIN-5526 fix unique queue async query count more than setting
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4a889f08ab94df6bccb5743cdc51d32a5d1b2b66
Author: Jiawei Li <10...@qq.com>
AuthorDate: Thu Feb 16 17:37:19 2023 +0800
KYLIN-5526 fix unique queue async query count more than setting
* KYLIN-5526 add check when add unique queue async query count
---
.../apache/kylin/rest/controller/NAsyncQueryController.java | 8 ++++++--
.../java/org/apache/kylin/rest/service/QueryService.java | 13 +++++--------
.../org/apache/kylin/rest/util/AsyncQueryRequestLimits.java | 8 +++++++-
3 files changed, 18 insertions(+), 11 deletions(-)
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java
index 8f84c63ddf..99a3f4ee7a 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java
@@ -125,10 +125,11 @@ public class NAsyncQueryController extends NBasicController {
if (StringUtils.isEmpty(sqlRequest.getSeparator())) {
sqlRequest.setSeparator(",");
}
+ AsyncQueryRequestLimits asyncQueryRequestLimits = null;
if (NProjectManager.getProjectConfig(sqlRequest.getProject()).isUniqueAsyncQueryYarnQueue()) {
- AsyncQueryRequestLimits.checkCount();
+ asyncQueryRequestLimits = new AsyncQueryRequestLimits();
}
-
+ AsyncQueryRequestLimits finalAsyncQueryRequestLimits = asyncQueryRequestLimits;
executorService.submit(Objects.requireNonNull(TtlRunnable.get(() -> {
String format = sqlRequest.getFormat().toLowerCase(Locale.ROOT);
String encode = sqlRequest.getEncode().toLowerCase(Locale.ROOT);
@@ -166,6 +167,9 @@ public class NAsyncQueryController extends NBasicController {
throw new RuntimeException(e1);
}
} finally {
+ if (finalAsyncQueryRequestLimits != null) {
+ finalAsyncQueryRequestLimits.close();
+ }
logger.info("Async query with queryId: {} end", queryContext.getQueryId());
QueryContext.current().close();
}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 846e7b6260..42fa8425b5 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -142,7 +142,6 @@ import org.apache.kylin.rest.response.TableMetaCacheResultV2;
import org.apache.kylin.rest.security.MutableAclRecord;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
-import org.apache.kylin.rest.util.AsyncQueryRequestLimits;
import org.apache.kylin.rest.util.PrepareSQLUtils;
import org.apache.kylin.rest.util.QueryCacheSignatureUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
@@ -304,13 +303,11 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
if (StringUtils.isNotEmpty(sqlRequest.getSparkQueue())) {
queryParams.setSparkQueue(sqlRequest.getSparkQueue());
}
- try (AsyncQueryRequestLimits ignored = new AsyncQueryRequestLimits()) {
- AsyncQueryJob asyncQueryJob = new AsyncQueryJob();
- asyncQueryJob.setProject(queryParams.getProject());
- asyncQueryJob.submit(queryParams);
- return buildSqlResponse(false, Collections.emptyList(), 0, Lists.newArrayList(),
- sqlRequest.getProject());
- }
+ AsyncQueryJob asyncQueryJob = new AsyncQueryJob();
+ asyncQueryJob.setProject(queryParams.getProject());
+ asyncQueryJob.submit(queryParams);
+ return buildSqlResponse(false, Collections.emptyList(), 0, Lists.newArrayList(),
+ sqlRequest.getProject());
}
SQLResponse fakeResponse = TableauInterceptor.tableauIntercept(queryParams.getSql());
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/util/AsyncQueryRequestLimits.java b/src/query-service/src/main/java/org/apache/kylin/rest/util/AsyncQueryRequestLimits.java
index ea3df6e9e4..382635f2aa 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/util/AsyncQueryRequestLimits.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/util/AsyncQueryRequestLimits.java
@@ -25,17 +25,23 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AsyncQueryRequestLimits implements AutoCloseable {
+ private static final Logger logger = LoggerFactory.getLogger(AsyncQueryRequestLimits.class);
+
private static volatile AtomicInteger asyncQueryCount = new AtomicInteger(0);
private static final int MAX_COUNT = KylinConfig.getInstanceFromEnv().getAsyncQueryMaxConcurrentJobs();
- private static void openAsyncQueryRequest() {
+ private static synchronized void openAsyncQueryRequest() {
if (MAX_COUNT <= 0) {
return;
}
+ checkCount();
asyncQueryCount.incrementAndGet();
+ logger.debug("current async query job count is {}.", asyncQueryCount.get());
}
[kylin] 30/38: KYLIN-5523 [FOLLOW UP] Fix using a computed column as the join key leads to a schema change failure
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit fb90074de24f60f7134c18da896e707e71d3241b
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Thu Mar 2 15:03:34 2023 +0800
KYLIN-5523 [FOLLOW UP] Fix using a computed column as the join key leads to a schema change failure
KYLIN-5523 [FOLLOW UP] expression of computed column change will be treated as a significant change to the model
KYLIN-5523 [FOLLOW UP] Table reloading cannot be affected by a damaged join key of computed column
---
.../java/org/apache/kylin/metadata/model/ColumnDesc.java | 12 +++++++++++-
.../test/java/org/apache/kylin/event/SchemaChangeTest.java | 5 +----
.../java/org/apache/kylin/rest/service/ModelService.java | 7 ++++++-
.../main/java/org/apache/kylin/query/schema/OLAPTable.java | 4 ++--
4 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 49094bf6a3..71cf9fedc4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -272,7 +272,17 @@ public class ColumnDesc implements Serializable {
} else if (!name.equals(other.name))
return false;
- return table == null ? other.table == null : table.getIdentity().equals(other.table.getIdentity());
+ boolean isSameTable = table == null ? other.table == null
+ : table.getIdentity().equals(other.table.getIdentity());
+ if (!isSameTable) {
+ return false;
+ }
+ if (this.isComputedColumn() && other.isComputedColumn()) {
+ // add a simple check of computed column's inner expression
+ return StringUtils.equals(computedColumnExpr, other.getComputedColumnExpr());
+ }
+
+ return !this.isComputedColumn() && !other.isComputedColumn();
}
@Override
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/event/SchemaChangeTest.java b/src/kylin-it/src/test/java/org/apache/kylin/event/SchemaChangeTest.java
index 831bf2f6dd..00b58e7418 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/event/SchemaChangeTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/event/SchemaChangeTest.java
@@ -79,7 +79,6 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
@@ -94,7 +93,6 @@ import lombok.val;
import lombok.var;
import lombok.extern.slf4j.Slf4j;
-@Ignore("disable unstable test")
@Slf4j
public class SchemaChangeTest extends AbstractMVCIntegrationTestCase {
@@ -246,7 +244,6 @@ public class SchemaChangeTest extends AbstractMVCIntegrationTestCase {
assertSqls();
}
- @Ignore("TODO: remove or adapt")
@Test
public void testChangeColumnType() throws Exception {
changeColumns(TABLE_IDENTITY, Sets.newHashSet("SRC_ID"), columnDesc -> columnDesc.setDatatype("string"));
@@ -334,7 +331,7 @@ public class SchemaChangeTest extends AbstractMVCIntegrationTestCase {
private void setupPushdownEnv() throws Exception {
getTestConfig().setProperty("kylin.query.pushdown.runner-class-name",
- "io.kyligence.kap.query.pushdown.PushDownRunnerJdbcImpl");
+ "org.apache.kylin.query.pushdown.PushDownRunnerJdbcImpl");
getTestConfig().setProperty("kylin.query.pushdown-enabled", "true");
// Load H2 Tables (inner join)
Connection h2Connection = DriverManager.getConnection("jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1", "sa", "");
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 21d52c7956..d0073de148 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -128,6 +128,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.common.util.ThreadUtil;
import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil;
import org.apache.kylin.job.SecondStorageJobParamUtil;
import org.apache.kylin.job.common.SegmentUtil;
@@ -3264,7 +3265,11 @@ public class ModelService extends AbstractModelService implements TableModelSupp
val modelManager = getManager(NDataModelManager.class, project);
val origin = modelManager.getDataModelDesc(modelRequest.getUuid());
val copyModel = modelManager.copyForWrite(origin);
- semanticUpdater.updateModelColumns(copyModel, modelRequest);
+ try {
+ semanticUpdater.updateModelColumns(copyModel, modelRequest);
+ } catch (Exception e) {
+ log.warn("Update existing model failed.{}", ThreadUtil.getKylinStackTrace());
+ }
copyModel.setBrokenReason(NDataModel.BrokenReason.SCHEMA);
copyModel.getAllNamedColumns().forEach(namedColumn -> {
if (columnIds.contains(namedColumn.getId())) {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index ff3ba69038..5b19ad4d4f 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,7 +19,7 @@
package org.apache.kylin.query.schema;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -249,7 +249,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
}
}
- Collections.sort(tableColumns, (o1, o2) -> o1.getZeroBasedIndex() - o2.getZeroBasedIndex());
+ tableColumns.sort(Comparator.comparingInt(ColumnDesc::getZeroBasedIndex));
return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
}
[kylin] 06/38: mirror: fix complie
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2a152803bb7186b89e314b4023e78b0a16ddd4b4
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Tue Apr 25 18:20:09 2023 +0800
mirror: fix complie
---
.../main/java/org/apache/kylin/rest/service/JobService.java | 1 -
.../java/org/apache/kylin/rest/service/SnapshotService.java | 13 -------------
.../org/apache/kylin/rest/service/ModelSemanticHelper.java | 1 -
.../org/apache/kylin/rest/service/ModelServiceTest.java | 1 -
.../apache/kylin/rest/service/ModelServiceQueryTest.java | 1 -
.../rest/service/ModelServiceWithSecondStorageTest.java | 1 -
.../src/test/scala/org/apache/kylin/common/SSSource.scala | 5 ++---
7 files changed, 2 insertions(+), 21 deletions(-)
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index 0e0c4a6e9f..eb668435fa 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -140,7 +140,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import io.kyligence.kap.metadata.epoch.EpochManager;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.Getter;
import lombok.Setter;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
index 1a40feb5ec..40a4b767a8 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
@@ -96,19 +96,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
@Component("snapshotService")
public class SnapshotService extends BasicService implements SnapshotSupporter {
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index 147b413710..df14faaf5a 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -96,7 +96,6 @@ import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinDesc;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.request.ModelRequest;
import org.apache.kylin.rest.response.BuildIndexResponse;
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index cb82fc5921..5b2926052f 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -158,7 +158,6 @@ import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.metadata.user.ManagedUser;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
index 2b0eeb31d8..351d675346 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
@@ -37,7 +37,6 @@ import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
diff --git a/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java b/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
index afe9add1cd..dd6c03c21e 100644
--- a/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
+++ b/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
@@ -38,7 +38,6 @@ import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
diff --git a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala
index dd25fd05f5..63b6414256 100644
--- a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala
+++ b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala
@@ -21,7 +21,6 @@ import java.io.{DataInputStream, File}
import java.nio.charset.Charset
import java.nio.file.Files
import java.util.Locale
-
import com.google.common.base.Preconditions
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils
@@ -30,7 +29,7 @@ import org.apache.kylin.common.util.TempMetadataBuilder
import org.apache.kylin.metadata.cube.model.{IndexPlan, NDataflowManager, NIndexPlanManager}
import org.apache.kylin.metadata.model.{NDataModel, NDataModelManager, NTableMetadataManager}
import org.apache.kylin.metadata.project.NProjectManager
-import org.apache.kylin.query.util.{QueryParams, QueryUtil}
+import org.apache.kylin.query.util.{PushDownUtil, QueryParams}
import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession}
import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.scalatest.Suite
@@ -84,7 +83,7 @@ trait SSSource extends SharedSparkSession with LocalMetadata {
.replaceAll("\"DEFAULT\"\\.", "")
val queryParams = new QueryParams("default", sqlForSpark, "DEFAULT", false)
queryParams.setKylinConfig(NProjectManager.getProjectConfig("default"))
- QueryUtil.massagePushDownSql(queryParams)
+ PushDownUtil.massagePushDownSql(queryParams)
}
def addModels(resourcePath: String, modelIds: Seq[String]): Unit = {
[kylin] 26/38: KYLIN-5533 [FOLLOW UP] fix BeanUtils copyProperties NPE
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c8a464d2cadc72f56fe51d98f2914eecdfe00e22
Author: lixiang <44...@qq.com>
AuthorDate: Tue Feb 28 17:56:52 2023 +0800
KYLIN-5533 [FOLLOW UP] fix BeanUtils copyProperties NPE
---
.../src/main/java/org/apache/kylin/rest/service/ModelService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index c3645f4af8..21d52c7956 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -695,7 +695,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
try {
NDataModelResponse nDataModelResponse = convertToDataModelResponse(dataModel, projectName, dfManager,
status, queryElem.isOnlyNormalDim());
- if (lite) {
+ if (lite && nDataModelResponse != null) {
return new NDataModelLiteResponse(nDataModelResponse, dataModel);
} else {
return nDataModelResponse;
[kylin] 10/38: KYLIN-5523 [FOLLOW UP] fix some function of cc as join key or filter column
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 5657afc7f2a827419d8b8beda4b294c75f6245e0
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Wed Feb 15 21:35:56 2023 +0800
KYLIN-5523 [FOLLOW UP] fix some function of cc as join key or filter column
---
.../org/apache/kylin/common/KylinConfigBase.java | 7 +-
.../kylin-backward-compatibility.properties | 1 +
.../org/apache/kylin/job/JoinedFlatTableTest.java | 242 --------------
.../kylin/metadata/model/JoinedFlatTable.java | 371 ---------------------
.../apache/kylin/rest/service/ModelService.java | 53 ++-
.../kylin/rest/service/ModelServiceTest.java | 32 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 59 +++-
.../apache/kylin/query/util/QueryAliasMatcher.java | 25 +-
.../apache/kylin/query/util/CCOnRealModelTest.java | 11 +-
.../apache/kylin/query/util/PushDownUtilTest.java | 171 +++++++++-
10 files changed, 282 insertions(+), 690 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 0d4796e6e3..f20dca320b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2502,6 +2502,7 @@ public abstract class KylinConfigBase implements Serializable {
public int getCalciteBindableCacheSize() {
return Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.maxSize", "10"));
}
+
public int getCalciteBindableCacheConcurrencyLevel() {
return Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.concurrencyLevel", "5"));
}
@@ -3456,7 +3457,7 @@ public abstract class KylinConfigBase implements Serializable {
}
public boolean skipCheckFlatTable() {
- return Boolean.parseBoolean(getOptional("kylin.model.skip-flattable-check", FALSE));
+ return Boolean.parseBoolean(getOptional("kylin.model.skip-check-flattable", FALSE));
}
public boolean isQueryExceptionCacheEnabled() {
@@ -3570,10 +3571,6 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.table.fast-reload-enabled", TRUE));
}
- public boolean isSkipCheckFlatTable() {
- return Boolean.parseBoolean(getOptional("kylin.model.skip-check-flattable", FALSE));
- }
-
public boolean isUnitOfWorkSimulationEnabled() {
return Boolean.parseBoolean(getOptional("kylin.env.unitofwork-simulation-enabled", FALSE));
}
diff --git a/src/core-common/src/main/resources/kylin-backward-compatibility.properties b/src/core-common/src/main/resources/kylin-backward-compatibility.properties
index d14394d8dd..fc239ae6a8 100644
--- a/src/core-common/src/main/resources/kylin-backward-compatibility.properties
+++ b/src/core-common/src/main/resources/kylin-backward-compatibility.properties
@@ -28,6 +28,7 @@ kylin.realization.providers=kylin.metadata.realization-providers
kylin.cube.dimension.customEncodingFactories=kylin.metadata.custom-dimension-encodings
kylin.cube.measure.customMeasureType.=kylin.metadata.custom-measure-types.
kap.metadata.semi-automatic-mode=kylin.metadata.semi-automatic-mode
+kylin.model.skip-flattable-check=kylin.model.skip-check-flattable
### Dictionary ###
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java b/src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
deleted file mode 100644
index 52ab091c31..0000000000
--- a/src/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.util.List;
-
-import org.apache.calcite.avatica.util.Quoting;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableRef;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.JoinedFlatTable;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.assertj.core.util.Lists;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableBiMap;
-
-public class JoinedFlatTableTest {
-
- private static final String QUOTE = Quoting.DOUBLE_QUOTE.string;
- private NDataModel dataModel = new NDataModel();
-
- @Before
- public void setUp() {
- dataModel.setUuid(RandomUtil.randomUUIDStr());
-
- TableDesc lineOrderTableDesc = new TableDesc();
- lineOrderTableDesc.setUuid("665a66d6-08d6-42b8-9be8-d9a0456ee250");
- lineOrderTableDesc.setName("LINEORDER");
- lineOrderTableDesc.setDatabase("SSB");
- lineOrderTableDesc.setSourceType(9);
- lineOrderTableDesc.setTableType("MANAGED");
-
- ColumnDesc loSuppkeyColDesc = new ColumnDesc();
- loSuppkeyColDesc.setId("1");
- loSuppkeyColDesc.setName("LO_SUPPKEY");
- loSuppkeyColDesc.setDatatype("integer");
- loSuppkeyColDesc.setComment("null");
- loSuppkeyColDesc.setTable(lineOrderTableDesc);
-
- ColumnDesc loTaxColDesc = new ColumnDesc();
- loTaxColDesc.setId("2");
- loTaxColDesc.setName("LO_TAX");
- loTaxColDesc.setDatatype("bigint");
- loTaxColDesc.setComment("null");
- loTaxColDesc.setTable(lineOrderTableDesc);
-
- ColumnDesc loRevenueColDesc = new ColumnDesc();
- loRevenueColDesc.setId("3");
- loRevenueColDesc.setName("LO_REVENUE");
- loRevenueColDesc.setDatatype("bigint");
- loRevenueColDesc.setComment("null");
- loRevenueColDesc.setTable(lineOrderTableDesc);
-
- ColumnDesc cc = new ColumnDesc("4", "PROFIT", "bigint", null, null, null,
- "case when `LINEORDER`.`LO_REVENUE` - `LINEORDER`.`LO_TAX` > 0 then 'LINEORDER' else null end");
- cc.setTable(lineOrderTableDesc);
-
- lineOrderTableDesc.setColumns(new ColumnDesc[] { loSuppkeyColDesc, loTaxColDesc, loRevenueColDesc, cc });
- TableRef lineOrderTableRef = new TableRef(dataModel, "LINEORDER", lineOrderTableDesc, false);
-
- TableDesc supplierTableDesc = new TableDesc();
- supplierTableDesc.setUuid("719e9bf4-82de-40ec-9454-ab43ff94eef4");
- supplierTableDesc.setName("SUPPLIER");
- supplierTableDesc.setDatabase("SSB");
- supplierTableDesc.setSourceType(9);
- supplierTableDesc.setTableType("MANAGED");
-
- ColumnDesc sSuppkey = new ColumnDesc();
- sSuppkey.setId("1");
- sSuppkey.setName("S_SUPPKEY");
- sSuppkey.setDatatype("integer");
- sSuppkey.setComment("null");
- sSuppkey.setTable(supplierTableDesc);
-
- ColumnDesc sCity = new ColumnDesc();
- sCity.setId("2");
- sCity.setName("S_CITY");
- sCity.setDatatype("varchar(4096)");
- sCity.setComment("null");
- sCity.setTable(supplierTableDesc);
-
- supplierTableDesc.setColumns(new ColumnDesc[] { sSuppkey, sCity });
- TableRef supplierTableRef = new TableRef(dataModel, "SUPPLIER", supplierTableDesc, false);
-
- ImmutableBiMap.Builder<Integer, TblColRef> effectiveCols = ImmutableBiMap.builder();
- effectiveCols.put(1, lineOrderTableRef.getColumn("LO_SUPPKEY"));
- effectiveCols.put(2, lineOrderTableRef.getColumn("LO_REVENUE"));
- effectiveCols.put(3, lineOrderTableRef.getColumn("LO_TAX"));
- effectiveCols.put(4, lineOrderTableRef.getColumn("PROFIT"));
- effectiveCols.put(5, supplierTableRef.getColumn("S_SUPPKEY"));
- effectiveCols.put(6, supplierTableRef.getColumn("S_CITY"));
- dataModel.setEffectiveCols(effectiveCols.build());
-
- dataModel.setRootFactTableName("SSB.LINEORDER");
- dataModel.setRootFactTableRef(lineOrderTableRef);
-
- JoinDesc joinDesc = new JoinDesc();
- joinDesc.setType("LEFT");
- joinDesc.setPrimaryKey(new String[] { "LINEORDER.LO_SUPPKEY" });
- joinDesc.setForeignKey(new String[] { "SUPPLIER.S_SUPPKEY" });
- joinDesc.setPrimaryTableRef(lineOrderTableRef);
- joinDesc.setPrimaryKeyColumns(new TblColRef[] { new TblColRef(lineOrderTableRef, loSuppkeyColDesc) });
- joinDesc.setForeignKeyColumns(new TblColRef[] { new TblColRef(supplierTableRef, sSuppkey) });
- JoinTableDesc supplierJoinTableDesc = new JoinTableDesc();
- supplierJoinTableDesc.setTable("SSB.SUPPLIER");
- supplierJoinTableDesc.setAlias("SUPPLIER");
- supplierJoinTableDesc.setKind(NDataModel.TableKind.LOOKUP);
- supplierJoinTableDesc.setTableRef(supplierTableRef);
- supplierJoinTableDesc.setJoin(joinDesc);
-
- dataModel.setJoinTables(Lists.newArrayList(supplierJoinTableDesc));
- dataModel.setFilterCondition("SUPPLIER.S_CITY != 'beijing'");
- ComputedColumnDesc cc1 = new ComputedColumnDesc();
- cc1.setExpression(
- "case when \"LINEORDER\".\"LO_REVENUE\" - \"LINEORDER\".\"LO_TAX\" > 0 then 'LINEORDER' else null end");
- cc1.setInnerExpression(
- "case when `LINEORDER`.`LO_REVENUE` - `LINEORDER`.`LO_TAX` > 0 then 'LINEORDER' else null end");
- cc1.setColumnName("PROFIT");
- cc1.setDatatype("bigint");
- dataModel.setComputedColumnDescs(Lists.newArrayList(cc1));
- }
-
- @Test
- public void testQuoteIdentifierInSqlExpr() {
- String cc = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel, "LINEORDER.LO_REVENUE-LINEORDER.LO_TAX", QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\"-\"LINEORDER\".\"LO_TAX\"", cc);
-
- String where1 = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel, "LINEORDER.LO_REVENUE-LINEORDER.LO_TAX>0",
- QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\"-\"LINEORDER\".\"LO_TAX\">0", where1);
-
- String where2 = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel,
- "LINEORDER.LO_REVENUE>100 AND LINEORDER.LO_TAX>0", QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\">100 AND \"LINEORDER\".\"LO_TAX\">0", where2);
-
- String where3 = JoinedFlatTable.quoteIdentifierInSqlExpr(dataModel,
- "`LINEORDER`.`LO_REVENUE`>100 AND `LINEORDER`.`LO_TAX`>0", QUOTE);
- Assert.assertEquals("\"LINEORDER\".\"LO_REVENUE\">100 AND \"LINEORDER\".\"LO_TAX\">0", where3);
- }
-
- @Test
- public void testGenerateSelectDataStatement() {
- String flatTableSql = JoinedFlatTable.generateSelectDataStatement(dataModel, false);
- String expectedSql = "SELECT \n" //
- + "\"LINEORDER\".\"LO_SUPPKEY\" as \"LINEORDER_LO_SUPPKEY\",\n"
- + "\"LINEORDER\".\"LO_REVENUE\" as \"LINEORDER_LO_REVENUE\",\n"
- + "\"LINEORDER\".\"LO_TAX\" as \"LINEORDER_LO_TAX\",\n"
- + "\"SUPPLIER\".\"S_SUPPKEY\" as \"SUPPLIER_S_SUPPKEY\",\n"
- + "\"SUPPLIER\".\"S_CITY\" as \"SUPPLIER_S_CITY\"\n" //
- + "FROM \n" //
- + "\"SSB\".\"LINEORDER\" as \"LINEORDER\" \n" //
- + "LEFT JOIN \"SSB\".\"SUPPLIER\" as \"SUPPLIER\"\n" //
- + "ON \"SUPPLIER\".\"S_SUPPKEY\"=\"LINEORDER\".\"LO_SUPPKEY\"\n" //
- + "WHERE \n" //
- + "1 = 1\n" //
- + " AND (\"SUPPLIER\".\"S_CITY\" != 'beijing')";
- Assert.assertEquals(expectedSql, flatTableSql.trim());
-
- NonEquiJoinCondition nonEquiJoinCondition = new NonEquiJoinCondition();
- nonEquiJoinCondition.setExpr("SUPPLIER.S_SUPPKEY <> LINEORDER.LO_SUPPKEY AND LINEORDER.LO_SUPPKEY > 10");
- dataModel.getJoinTables().get(0).getJoin().setNonEquiJoinCondition(nonEquiJoinCondition);
- String nonEquiJoinConditionSql = JoinedFlatTable.generateSelectDataStatement(dataModel, false);
- Assert.assertTrue(nonEquiJoinConditionSql.contains(
- "ON \"SUPPLIER\".\"S_SUPPKEY\" <> \"LINEORDER\".\"LO_SUPPKEY\" AND \"LINEORDER\".\"LO_SUPPKEY\" > 10"));
- dataModel.getJoinTables().get(0).getJoin().setNonEquiJoinCondition(null);
- }
-
- @Test
- public void testQuoteIdentifier() {
- List<String> tablePatterns = JoinedFlatTable.getTableNameOrAliasPatterns("KYLIN_SALES");
- String exprTable = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
- String expectedExprTable = "\"KYLIN_SALES\".PRICE * \"KYLIN_SALES\".COUNT";
- String quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- exprTable = "`KYLIN_SALES`.PRICE * KYLIN_SALES.COUNT";
- expectedExprTable = "\"KYLIN_SALES\".PRICE * \"KYLIN_SALES\".COUNT";
- quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- exprTable = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- expectedExprTable = "\"KYLIN_SALES\".PRICE AS KYLIN_SALES_PRICE * \"KYLIN_SALES\".COUNT AS KYLIN_SALES_COUNT";
- quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- exprTable = "(KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
- expectedExprTable = "(\"KYLIN_SALES\".PRICE AS KYLIN_SALES_PRICE > 1 and \"KYLIN_SALES\".COUNT AS KYLIN_SALES_COUNT > 50)";
- quotedExprTable = JoinedFlatTable.quoteIdentifier(exprTable, QUOTE, "KYLIN_SALES", tablePatterns);
- Assert.assertEquals(expectedExprTable, quotedExprTable);
-
- List<String> columnPatterns = JoinedFlatTable.getColumnNameOrAliasPatterns("PRICE");
- String expr = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
- String expectedExpr = "KYLIN_SALES.\"PRICE\" * KYLIN_SALES.COUNT";
- String quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "KYLIN_SALES.PRICE / KYLIN_SALES.COUNT";
- expectedExpr = "KYLIN_SALES.\"PRICE\" / KYLIN_SALES.COUNT";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- expectedExpr = "KYLIN_SALES.\"PRICE\" AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "(PRICE > 1 AND COUNT > 50)";
- expectedExpr = "(\"PRICE\" > 1 AND COUNT > 50)";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
-
- expr = "PRICE>1 and `PRICE` < 15";
- expectedExpr = "\"PRICE\">1 and \"PRICE\" < 15";
- quotedExpr = JoinedFlatTable.quoteIdentifier(expr, QUOTE, "PRICE", columnPatterns);
- Assert.assertEquals(expectedExpr, quotedExpr);
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
deleted file mode 100644
index b222252d63..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
+++ /dev/null
@@ -1,371 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.calcite.avatica.util.Quoting;
-import org.apache.commons.lang3.StringUtils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class JoinedFlatTable {
-
- private static final String DATABASE_AND_TABLE = "%s.%s";
-
- private static final String QUOTE = Quoting.DOUBLE_QUOTE.string;
- private static final String UNDER_LINE = "_";
- private static final String DOT = ".";
- private static final Pattern BACK_TICK_DOT_PATTERN = Pattern
- .compile("`[^\\f\\n\\r\\t\\v]+?`\\.`[^\\f\\n\\r\\t\\v]+?`");
-
- private JoinedFlatTable() {
- }
-
- private static String quote(String identifier) {
- return QUOTE + identifier + QUOTE;
- }
-
- private static String colName(TblColRef col) {
- return col.getTableAlias() + UNDER_LINE + col.getName();
- }
-
- private static String quotedTable(TableDesc table) {
- if (table.getCaseSensitiveDatabase().equals("null")) {
- return quote(table.getCaseSensitiveName().toUpperCase(Locale.ROOT));
- }
- return String.format(Locale.ROOT, DATABASE_AND_TABLE,
- quote(table.getCaseSensitiveDatabase().toUpperCase(Locale.ROOT)),
- quote(table.getCaseSensitiveName().toUpperCase(Locale.ROOT)));
- }
-
- private static String quotedColExpressionInSourceDB(NDataModel modelDesc, TblColRef col) {
- Map<String, ComputedColumnDesc> ccMap = Maps.newHashMap();
- modelDesc.getComputedColumnDescs().forEach(cc -> ccMap.putIfAbsent(cc.getColumnName(), cc));
- if (!col.getColumnDesc().isComputedColumn()) {
- return quote(col.getTableAlias()) + DOT + quote(col.getName());
- }
- return quoteIdentifierInSqlExpr(modelDesc, ccMap.get(col.getName()).getInnerExpression(), QUOTE);
- }
-
- private static String appendEffectiveColumnsStatement(NDataModel modelDesc, List<TblColRef> effectiveColumns,
- boolean singleLine, boolean includeCC, Function<TblColRef, String> namingFunction) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder subSql = new StringBuilder();
- if (effectiveColumns.isEmpty()) {
- subSql.append("1");
- return subSql.toString();
- }
-
- effectiveColumns.forEach(col -> {
- if (includeCC || !col.getColumnDesc().isComputedColumn()) {
- if (subSql.length() > 0) {
- subSql.append(",").append(sep);
- }
-
- String colName = namingFunction != null ? namingFunction.apply(col) : colName(col);
- subSql.append(quotedColExpressionInSourceDB(modelDesc, col)).append(" as ").append(quote(colName));
- }
- });
-
- return subSql.toString();
- }
-
- private static String appendWhereStatement(NDataModel modelDesc, boolean singleLine) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder whereBuilder = new StringBuilder();
- whereBuilder.append("1 = 1").append(sep);
-
- if (StringUtils.isNotEmpty(modelDesc.getFilterCondition())) {
- String quotedFilterCondition = quoteIdentifierInSqlExpr(modelDesc, modelDesc.getFilterCondition(), QUOTE);
- whereBuilder.append(" AND (").append(quotedFilterCondition).append(") ").append(sep); // -> filter condition contains special character may cause bug
- }
-
- return whereBuilder.toString();
- }
-
- private static String appendJoinStatement(NDataModel modelDesc, boolean singleLine) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder subSql = new StringBuilder();
-
- Set<TableRef> dimTableCache = new HashSet<>();
- TableRef rootTable = modelDesc.getRootFactTable();
- subSql.append(quotedTable(modelDesc.getRootFactTable().getTableDesc())).append(" as ")
- .append(quote(rootTable.getAlias())).append(" ").append(sep);
-
- for (JoinTableDesc lookupDesc : modelDesc.getJoinTables()) {
- JoinDesc join = lookupDesc.getJoin();
- if (checkJoinDesc(join)) {
- continue;
- }
- String joinType = join.getType().toUpperCase(Locale.ROOT);
- TableRef dimTable = lookupDesc.getTableRef();
- if (dimTableCache.contains(dimTable)) {
- continue;
- }
- subSql.append(joinType).append(" JOIN ").append(quotedTable(dimTable.getTableDesc())).append(" as ")
- .append(quote(dimTable.getAlias())).append(sep);
- subSql.append("ON ");
-
- if (Objects.nonNull(join.getNonEquiJoinCondition())) {
- subSql.append(quoteIdentifierInSqlExpr(modelDesc, join.getNonEquiJoinCondition().getExpr(), QUOTE));
- } else {
- TblColRef[] pk = join.getPrimaryKeyColumns();
- TblColRef[] fk = join.getForeignKeyColumns();
- if (pk.length != fk.length) {
- throw new RuntimeException(
- String.format(Locale.ROOT, "Invalid join condition of lookup table: %s", lookupDesc));
- }
-
- for (int i = 0; i < pk.length; i++) {
- if (i > 0) {
- subSql.append(" AND ");
- }
- subSql.append(quotedColExpressionInSourceDB(modelDesc, fk[i])).append("=")
- .append(quotedColExpressionInSourceDB(modelDesc, pk[i]));
- }
- }
- subSql.append(sep);
- dimTableCache.add(dimTable);
- }
- return subSql.toString();
- }
-
- private static String getSepBySingleLineTag(boolean singleLine) {
- return singleLine ? " " : "\n";
- }
-
- public static String generateSelectDataStatement(NDataModel modelDesc, boolean singleLine) {
- return generateSelectDataStatement(modelDesc, Lists.newArrayList(modelDesc.getEffectiveCols().values()),
- singleLine, false, true, null);
- }
-
- public static String generateSelectDataStatement(NDataModel modelDesc, List<TblColRef> effectiveColumns,
- boolean singleLine, boolean includeCC, boolean includeFilter, Function<TblColRef, String> namingFunction) {
- final String sep = getSepBySingleLineTag(singleLine);
-
- StringBuilder sql = new StringBuilder("SELECT ").append(sep);
- String columnsStatement = appendEffectiveColumnsStatement(modelDesc, effectiveColumns, singleLine, includeCC,
- namingFunction);
- sql.append(columnsStatement.endsWith(sep) ? columnsStatement : columnsStatement + sep);
- sql.append("FROM ").append(sep);
- String joinStatement = appendJoinStatement(modelDesc, singleLine);
- sql.append(joinStatement.endsWith(sep) ? joinStatement : joinStatement + sep);
- if (includeFilter) {
- sql.append("WHERE ").append(sep);
- sql.append(appendWhereStatement(modelDesc, singleLine));
- }
- return sql.toString();
- }
-
- private static boolean checkJoinDesc(JoinDesc join) {
- return join == null || join.getType().equals("");
- }
-
- private static String getColumnAlias(String tableName, String columnName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
- if (!colToAliasMap.containsKey(columnName)) {
- return null;
- }
- return colToAliasMap.get(columnName);
- }
-
- private static boolean columnHasAlias(String tableName, String columnName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
- return colToAliasMap.containsKey(columnName);
- }
-
- private static Map<String, String> getColToColAliasMapInTable(String tableName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- if (tableToColumnsMap.containsKey(tableName)) {
- return tableToColumnsMap.get(tableName);
- }
- return Maps.newHashMap();
- }
-
- private static Set<String> listColumnsInTable(String tableName,
- Map<String, Map<String, String>> tableToColumnsMap) {
- Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
- return colToAliasMap.keySet();
- }
-
- @VisibleForTesting
- public static String quoteIdentifier(String sqlExpr, String quotation, String identifier,
- List<String> identifierPatterns) {
- String quotedIdentifier = quotation + identifier.trim() + quotation;
-
- for (String pattern : identifierPatterns) {
- Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.DOTALL).matcher(sqlExpr);
- if (matcher.find()) {
- sqlExpr = matcher.replaceAll("$1" + quotedIdentifier + "$3");
- }
- }
- return sqlExpr;
- }
-
- private static boolean isIdentifierNeedToQuote(String sqlExpr, String identifier, List<String> identifierPatterns) {
- if (StringUtils.isBlank(sqlExpr) || StringUtils.isBlank(identifier)) {
- return false;
- }
-
- for (String pattern : identifierPatterns) {
- if (Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.DOTALL).matcher(sqlExpr).find()) {
- return true;
- }
- }
- return false;
- }
-
- @VisibleForTesting
- public static List<String> getTableNameOrAliasPatterns(String tableName) {
- Preconditions.checkNotNull(tableName);
- // Pattern must contain these regex groups, and place identifier in sec group ($2)
- List<String> patterns = Lists.newArrayList();
- patterns.add("([+\\-*/%&|^=><\\s,(])(" + tableName.trim() + ")(\\.)");
- patterns.add("([+\\-*/%&|^=><\\s,(])(`" + tableName.trim() + "`)(\\.)");
- patterns.add("([\\.\\s])(" + tableName.trim() + ")([,\\s)])");
- patterns.add("([\\.\\s])(`" + tableName.trim() + "`)([,\\s)])");
- patterns.add("(^)(" + tableName.trim() + ")([\\.])");
- patterns.add("(^)(`" + tableName.trim() + "`)([\\.])");
- return patterns;
- }
-
- @VisibleForTesting
- public static List<String> getColumnNameOrAliasPatterns(String colName) {
- Preconditions.checkNotNull(colName);
- // Pattern must contain these regex groups, and place identifier in sec group ($2)
- List<String> patterns = Lists.newArrayList();
- patterns.add("([\\.\\s(])(" + colName.trim() + ")([+\\-*/%&|^=><\\s,)]|$)");
- patterns.add("([\\.\\s(])(`" + colName.trim() + "`)([+\\-*/%&|^=><\\s,)]|$)");
- patterns.add("(^)(" + colName.trim() + ")([+\\-*/%&|^=><\\s,)])");
- patterns.add("(^)(`" + colName.trim() + "`)([+\\-*/%&|^=><\\s,)])");
- return patterns;
- }
-
- private static Map<String, Map<String, String>> buildTableToColumnsMap(NDataModel modelDesc) {
- Map<String, Map<String, String>> map = Maps.newHashMap();
- Set<TblColRef> colRefs = modelDesc.getEffectiveCols().values();
- for (TblColRef colRef : colRefs) {
- String colName = colRef.getName();
- String tableName = colRef.getTableRef().getTableName();
- String colAlias = colRef.getTableAlias() + "_" + colRef.getName();
- if (map.containsKey(tableName)) {
- map.get(tableName).put(colName, colAlias);
- } else {
- Map<String, String> colToAliasMap = Maps.newHashMap();
- colToAliasMap.put(colName, colAlias);
- map.put(tableName, colToAliasMap);
- }
- }
- return map;
- }
-
- private static Map<String, String> buildTableToTableAliasMap(NDataModel modelDesc) {
- Map<String, String> map = Maps.newHashMap();
- Set<TblColRef> colRefs = modelDesc.getEffectiveCols().values();
- for (TblColRef colRef : colRefs) {
- String tableName = colRef.getTableRef().getTableName();
- String alias = colRef.getTableAlias();
- map.put(tableName, alias);
- }
- return map;
- }
-
- /**
- * Used for quote identifiers in Sql Filter Expression & Computed Column Expression for flat table
- * @param modelDesc
- * @param sqlExpr
- * @param quotation
- * @return
- */
- @VisibleForTesting
- public static String quoteIdentifierInSqlExpr(NDataModel modelDesc, String sqlExpr, String quotation) {
- if (BACK_TICK_DOT_PATTERN.matcher(sqlExpr).find()) {
- return quoteIdentifierInSqlBackTickExpr(sqlExpr, quotation);
- }
- Map<String, String> tabToAliasMap = buildTableToTableAliasMap(modelDesc);
- Map<String, Map<String, String>> tabToColsMap = buildTableToColumnsMap(modelDesc);
-
- boolean tableMatched = false;
- for (Map.Entry<String, String> tableEntry : tabToAliasMap.entrySet()) {
- List<String> tabPatterns = getTableNameOrAliasPatterns(tableEntry.getKey());
- if (isIdentifierNeedToQuote(sqlExpr, tableEntry.getKey(), tabPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, tableEntry.getKey(), tabPatterns);
- tableMatched = true;
- }
-
- String tabAlias = tableEntry.getValue();
- List<String> tabAliasPatterns = getTableNameOrAliasPatterns(tabAlias);
- if (isIdentifierNeedToQuote(sqlExpr, tabAlias, tabAliasPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, tabAlias, tabAliasPatterns);
- tableMatched = true;
- }
-
- if (!tableMatched) {
- continue;
- }
-
- Set<String> columns = listColumnsInTable(tableEntry.getKey(), tabToColsMap);
- for (String column : columns) {
- List<String> colPatterns = getColumnNameOrAliasPatterns(column);
- if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, column, colPatterns);
- }
- if (columnHasAlias(tableEntry.getKey(), column, tabToColsMap)) {
- String colAlias = getColumnAlias(tableEntry.getKey(), column, tabToColsMap);
- List<String> colAliasPattern = getColumnNameOrAliasPatterns(colAlias);
- if (isIdentifierNeedToQuote(sqlExpr, colAlias, colAliasPattern)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, colAlias, colPatterns);
- }
- }
- }
-
- tableMatched = false; //reset
- }
- return sqlExpr;
- }
-
- private static String quoteIdentifierInSqlBackTickExpr(String sqlExpr, String quotation) {
- String result = sqlExpr;
- Matcher matcher = BACK_TICK_DOT_PATTERN.matcher(sqlExpr);
- while (matcher.find()) {
- String target = matcher.group();
- target = target.replace("`", quotation);
- result = result.replace(matcher.group(), target);
- }
- return result;
- }
-
-}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index a7dd882b84..de7d4191f4 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -167,7 +167,6 @@ import org.apache.kylin.metadata.model.FusionModelManager;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.JoinedFlatTable;
import org.apache.kylin.metadata.model.ManagementType;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
@@ -1292,8 +1291,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public String getModelSql(String modelId, String project) {
aclEvaluate.checkProjectReadPermission(project);
- NDataModel modelDesc = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
- return JoinedFlatTable.generateSelectDataStatement(modelDesc, false);
+ NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
+ return PushDownUtil.generateFlatTableSql(model, project, false);
}
public List<RelatedModelResponse> getRelateModels(String project, String table, String modelId) {
@@ -1712,16 +1711,12 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
@VisibleForTesting
- public void checkFlatTableSql(NDataModel dataModel) {
- String project = dataModel.getProject();
- ProjectInstance prjInstance = getManager(NProjectManager.class).getProject(project);
- if (KylinConfig.getInstanceFromEnv().isUTEnv() || prjInstance.getConfig().isSkipCheckFlatTable()) {
- return;
- }
- if (getModelConfig(dataModel).skipCheckFlatTable()) {
+ public void checkFlatTableSql(NDataModel model) {
+ if (skipCheckFlatTable(model)) {
return;
}
- long rangePartitionTableCount = dataModel.getAllTableRefs().stream()
+
+ long rangePartitionTableCount = model.getAllTableRefs().stream()
.filter(p -> p.getTableDesc().isRangePartition()).count();
if (rangePartitionTableCount > 0) {
logger.info("Range partitioned tables do not support pushdown, so do not need to perform subsequent logic");
@@ -1729,22 +1724,34 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
try {
+ String project = model.getProject();
+ ProjectInstance prjInstance = getManager(NProjectManager.class).getProject(project);
if (prjInstance.getSourceType() == ISourceAware.ID_SPARK
- && dataModel.getModelType() == NDataModel.ModelType.BATCH) {
+ && model.getModelType() == NDataModel.ModelType.BATCH) {
SparkSession ss = SparderEnv.getSparkSession();
- String flatTableSql = JoinedFlatTable.generateSelectDataStatement(dataModel, false);
+ String flatTableSql = PushDownUtil.generateFlatTableSql(model, project, false);
QueryParams queryParams = new QueryParams(project, flatTableSql, "default", false);
queryParams.setKylinConfig(prjInstance.getConfig());
queryParams.setAclInfo(AclPermissionUtil.createAclInfo(project, getCurrentUserGroups()));
- String pushdownSql = PushDownUtil.massagePushDownSql(queryParams);
- ss.sql(pushdownSql);
+ ss.sql(PushDownUtil.massagePushDownSql(queryParams));
}
} catch (Exception e) {
- buildExceptionMessage(dataModel, e);
+ buildExceptionMessage(model, e);
+ }
+ }
+
+ private boolean skipCheckFlatTable(NDataModel model) {
+ if (KylinConfig.getInstanceFromEnv().isUTEnv()) {
+ return true;
}
+ IndexPlan indexPlan = getIndexPlan(model.getId(), model.getProject());
+ KylinConfig config = indexPlan == null || indexPlan.getConfig() == null
+ ? NProjectManager.getProjectConfig(model.getProject())
+ : indexPlan.getConfig();
+ return config.skipCheckFlatTable();
}
- private static void buildExceptionMessage(NDataModel dataModel, Exception e) {
+ private void buildExceptionMessage(NDataModel dataModel, Exception e) {
Pattern pattern = Pattern.compile("cannot resolve '(.*?)' given input columns");
Matcher matcher = pattern.matcher(e.getMessage().replace("`", ""));
if (matcher.find()) {
@@ -1761,14 +1768,6 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
}
- private KylinConfig getModelConfig(NDataModel dataModel) {
- IndexPlan indexPlan = getIndexPlan(dataModel.getId(), dataModel.getProject());
- if (indexPlan == null || indexPlan.getConfig() == null) {
- return getManager(NProjectManager.class).getProject(dataModel.getProject()).getConfig();
- }
- return indexPlan.getConfig();
- }
-
private void validatePartitionDateColumn(ModelRequest modelRequest) {
if (Objects.nonNull(modelRequest.getPartitionDesc())) {
if (StringUtils.isNotEmpty(modelRequest.getPartitionDesc().getPartitionDateColumn())) {
@@ -2350,7 +2349,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public JobInfoResponse fixSegmentHoles(String project, String modelId, List<SegmentTimeRequest> segmentHoles,
- Set<String> ignoredSnapshotTables) throws Exception {
+ Set<String> ignoredSnapshotTables) throws SQLException {
aclEvaluate.checkProjectOperationPermission(project);
NDataModel modelDesc = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
checkModelAndIndexManually(project, modelId);
@@ -2580,7 +2579,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
boolean buildSegmentOverlapEnable = getIndexPlan(model.getId(), project).getConfig()
.isBuildSegmentOverlapEnabled();
- boolean isBuildAllIndexesFinally = batchIndexIds == null || batchIndexIds.size() == 0
+ boolean isBuildAllIndexesFinally = CollectionUtils.isEmpty(batchIndexIds)
|| batchIndexIds.size() == getIndexPlan(model.getId(), project).getAllIndexes().size();
for (NDataSegment existedSegment : segments) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 5b2926052f..b69b89585b 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -84,8 +84,8 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -2203,7 +2203,6 @@ public class ModelServiceTest extends SourceTestCase {
request.setEnd("100");
request.setUuid(RandomUtil.randomUUIDStr());
modelService.createModel(request.getProject(), request);
- //TODO modelService.updateModelToResourceStore(deserialized, "default");
List<NDataModelResponse> dataModelDescs = modelService.getModels("nmodel_cc_test", "default", true, null, null,
"", false);
@@ -5161,20 +5160,23 @@ public class ModelServiceTest extends SourceTestCase {
public void testBuildExceptionMessage() {
NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
NDataModel dataModel = modelManager.getDataModelDesc("a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94");
- {
- val testException = new RuntimeException("test");
- Assert.assertThrows("model [test_encoding], Something went wrong. test", KylinException.class,
- () -> ReflectionTestUtils.invokeMethod(ModelService.class, "buildExceptionMessage", dataModel,
- testException));
- }
+ String toValidMethodName = "buildExceptionMessage";
+ String expectedMsg = "model [test_encoding], Something went wrong. test";
+ val testException = new RuntimeException("test");
+ Assert.assertThrows(expectedMsg, KylinException.class,
+ () -> ReflectionTestUtils.invokeMethod(modelService, toValidMethodName, dataModel, testException));
+ }
- {
- val testException = new RuntimeException("cannot resolve 'test' given input columns");
- Assert.assertThrows(
- "Can’t save model \"test_encoding\". Please ensure that the used column \"test\" exist in source table \"DEFAULT.TEST_ENCODING\".",
- KylinException.class, () -> ReflectionTestUtils.invokeMethod(ModelService.class,
- "buildExceptionMessage", dataModel, testException));
- }
+ @Test
+ public void testBuildExceptionMessageCausedByResolveProblem() {
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
+ NDataModel dataModel = modelManager.getDataModelDesc("a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94");
+ String toValidMethodName = "buildExceptionMessage";
+ String expectedMsg = "Can’t save model \"test_encoding\". Please ensure that the used column \"test\" "
+ + "exist in source table \"DEFAULT.TEST_ENCODING\".";
+ val testException = new RuntimeException("cannot resolve 'test' given input columns");
+ Assert.assertThrows(expectedMsg, KylinException.class,
+ () -> ReflectionTestUtils.invokeMethod(modelService, toValidMethodName, dataModel, testException));
}
@Test
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 9f3be8de77..b902bfe3f5 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -90,6 +90,7 @@ public class PushDownUtil {
.compile("/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/");
public static final String DEFAULT_SCHEMA = "DEFAULT";
private static final String CC_SPLITTER = "'##CC_PUSH_DOWN_TOKEN##'";
+ private static final String UNDER_LINE = "_";
private static final ExecutorService asyncExecutor = Executors.newCachedThreadPool();
private static final Map<String, IPushDownConverter> PUSH_DOWN_CONVERTER_MAP = Maps.newConcurrentMap();
@@ -109,11 +110,8 @@ public class PushDownUtil {
}
public static PushdownResult tryIterQuery(QueryParams queryParams) throws SQLException {
-
- String sql = queryParams.getSql();
- String project = queryParams.getProject();
-
KylinConfig projectConfig = NProjectManager.getProjectConfig(queryParams.getProject());
+ queryParams.setKylinConfig(projectConfig);
if (!projectConfig.isPushDownEnabled()) {
checkPushDownIncapable(queryParams);
return null;
@@ -130,21 +128,17 @@ public class PushDownUtil {
logger.info("Kylin cannot support non-select queries, routing to other engines");
}
+ // Set a push-down engine for query context.
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(projectConfig.getPushDownRunnerClassName());
- runner.init(projectConfig, project);
+ runner.init(projectConfig, queryParams.getProject());
logger.debug("Query Pushdown runner {}", runner);
-
- // set pushdown engine in query context
-
- // for file source
int sourceType = projectConfig.getDefaultSource();
String engine = sourceType == ISourceAware.ID_SPARK && KapConfig.getInstanceFromEnv().isCloud()
? QueryContext.PUSHDOWN_OBJECT_STORAGE
: runner.getName();
QueryContext.current().setPushdownEngine(engine);
- queryParams.setKylinConfig(projectConfig);
- queryParams.setSql(sql);
+ String sql;
try {
sql = massagePushDownSql(queryParams);
} catch (NoAuthorizedColsError e) {
@@ -154,13 +148,12 @@ public class PushDownUtil {
QueryContext.currentTrace().startSpan(QueryTrace.PREPARE_AND_SUBMIT_JOB);
if (queryParams.isSelect()) {
- PushdownResult result = runner.executeQueryToIterator(sql, project);
+ PushdownResult result = runner.executeQueryToIterator(sql, queryParams.getProject());
if (QueryContext.current().getQueryTagInfo().isAsyncQuery()) {
AsyncQueryUtil.saveMetaDataAndFileInfo(QueryContext.current(), result.getColumnMetas());
}
return result;
}
-
return PushdownResult.emptyResult();
}
@@ -189,6 +182,10 @@ public class PushDownUtil {
}
public static String massagePushDownSql(QueryParams queryParams) {
+ if (queryParams.getSql() == null) {
+ return StringUtils.EMPTY;
+ }
+
String sql = queryParams.getSql();
sql = QueryUtil.trimRightSemiColon(sql);
sql = SQL_HINT_PATTERN.matcher(sql).replaceAll("");
@@ -225,6 +222,41 @@ public class PushDownUtil {
return converters;
}
+ /**
+ * This method is currently only used for verifying the flat-table generated by the model.
+ */
+ public static String generateFlatTableSql(NDataModel model, String project, boolean singleLine) {
+ String sep = singleLine ? " " : "\n";
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("SELECT ").append(sep);
+
+ List<TblColRef> tblColRefs = Lists.newArrayList(model.getEffectiveCols().values());
+ if (tblColRefs.isEmpty()) {
+ sqlBuilder.append("1 ").append(sep);
+ } else {
+ String allColStr = tblColRefs.stream() //
+ .filter(colRef -> !colRef.getColumnDesc().isComputedColumn()) //
+ .map(colRef -> {
+ String s = colRef.getTableAlias() + UNDER_LINE + colRef.getName();
+ String colName = StringHelper.doubleQuote(s);
+ return colRef.getDoubleQuoteExp() + " as " + colName + sep;
+ }).collect(Collectors.joining(", "));
+ sqlBuilder.append(allColStr);
+ }
+
+ sqlBuilder.append("FROM ").append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity());
+ appendJoinStatement(model, sqlBuilder, singleLine);
+
+ sqlBuilder.append("WHERE ").append(sep);
+ sqlBuilder.append("1 = 1").append(sep);
+ if (StringUtils.isNotEmpty(model.getFilterCondition())) {
+ String filterCondition = massageExpression(model, project, model.getFilterCondition(), null);
+ sqlBuilder.append(" AND (").append(filterCondition).append(") ").append(sep);
+ }
+
+ return new EscapeTransformer().transform(sqlBuilder.toString());
+ }
+
public static String expandComputedColumnExp(NDataModel model, String project, String expression) {
StringBuilder forCC = new StringBuilder();
forCC.append("select ").append(expression).append(" ,").append(CC_SPLITTER) //
@@ -237,6 +269,7 @@ public class PushDownUtil {
// massage nested CC for drafted model
Map<String, NDataModel> modelMap = Maps.newHashMap();
modelMap.put(model.getUuid(), model);
+ ccSql = new EscapeTransformer().transform(ccSql);
ccSql = RestoreFromComputedColumn.convertWithGivenModels(ccSql, project, DEFAULT_SCHEMA, modelMap);
} catch (Exception e) {
logger.warn("Failed to massage SQL expression [{}] with input model {}", ccSql, model.getUuid(), e);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
index ab4fb87d2e..b9c1b579ea 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
@@ -43,7 +43,6 @@ import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -72,9 +71,11 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
// match alias in query to alias in model
// Not designed to reuse, re-new per query
+@Slf4j
public class QueryAliasMatcher {
static final ColumnRowType MODEL_VIEW_COLUMN_ROW_TYPE = new ColumnRowType(new ArrayList<>());
private static final ColumnRowType SUBQUERY_TAG = new ColumnRowType(null);
@@ -196,8 +197,7 @@ public class QueryAliasMatcher {
return null;
}
JoinsGraph joinsGraph = new JoinsGraph(firstTable, joinDescs);
- KylinConfigExt projectConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
- .getConfig();
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
if (sqlJoinCapturer.foundJoinOnCC) {
// 1st round: dry run without cc expr comparison to collect model alias matching
@@ -241,6 +241,7 @@ public class QueryAliasMatcher {
}
private static class CCJoinEdgeMatcher extends DefaultJoinEdgeMatcher {
+ private static final EscapeTransformer TRANSFORMER = new EscapeTransformer();
transient QueryAliasMatchInfo matchInfo;
boolean compareCCExpr;
@@ -263,12 +264,20 @@ public class QueryAliasMatcher {
|| (!a.isComputedColumn() && b.isComputedColumn())) {
return false;
} else {
- if (!compareCCExpr)
+ if (!compareCCExpr) {
return true;
-
- SqlNode node1 = CalciteParser.getExpNode(a.getComputedColumnExpr());
- SqlNode node2 = CalciteParser.getExpNode(b.getComputedColumnExpr());
- return ExpressionComparator.isNodeEqual(node1, node2, matchInfo, new AliasDeduceImpl(matchInfo));
+ }
+ try {
+ SqlNode node1 = CalciteParser.getExpNode(TRANSFORMER.transform(a.getDoubleQuoteInnerExpr()));
+ SqlNode node2 = CalciteParser.getExpNode(TRANSFORMER.transform(b.getDoubleQuoteInnerExpr()));
+ return ExpressionComparator.isNodeEqual(node1, node2, matchInfo, new AliasDeduceImpl(matchInfo));
+ } catch (Exception e) {
+ // If this situation occurs, it means that there is an error in the parsing of the computed column.
+ // Therefore, we can directly assume that these two computed columns are not equal.
+ log.error("Failed to parse expressions, {} or {}", a.getComputedColumnExpr(),
+ b.getComputedColumnExpr());
+ return false;
+ }
}
}
}
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
index 1b337e0569..658d266530 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
@@ -113,7 +113,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
// ignored for KAP#16258
- @Ignore
+ @Ignore("historic ignored")
@Test
public void testSubquery() {
{
@@ -198,7 +198,6 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- @Ignore("Not support CC on Join condition")
public void testJoinOnCC() {
{
String originSql = "select count(*) from TEST_KYLIN_FACT\n"
@@ -232,11 +231,9 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testNoFrom() throws Exception {
- String originSql = "select sum(price * item_count),(SELECT 1 as VERSION) from test_kylin_fact";
- String ccSql = "select sum(TEST_KYLIN_FACT.DEAL_AMOUNT),(SELECT 1 as VERSION) from test_kylin_fact";
-
- check(converter, originSql, ccSql);
+ public void testNoFrom() {
+ check(converter, "select sum(price * item_count),(SELECT 1 as VERSION) from test_kylin_fact",
+ "select sum(TEST_KYLIN_FACT.DEAL_AMOUNT),(SELECT 1 as VERSION) from test_kylin_fact");
}
@Test
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index 0a57502dcf..fdc7dcf28c 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -18,10 +18,22 @@
package org.apache.kylin.query.util;
+import java.util.List;
import java.util.Properties;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.exception.QueryErrorCode;
+import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -49,7 +61,8 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
PushDownUtil.tryIterQuery(queryParams);
Assert.fail();
} catch (Exception e) {
- Assert.assertFalse(e instanceof IllegalArgumentException);
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(ServerErrorCode.SPARK_FAILURE.toErrorCode(), ((KylinException) e).getErrorCode());
}
}
@@ -64,7 +77,9 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
PushDownUtil.tryIterQuery(queryParams);
Assert.fail();
} catch (Exception e) {
- Assert.assertFalse(e instanceof IllegalArgumentException);
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(QueryErrorCode.INVALID_PARAMETER_PUSH_DOWN.toErrorCode(),
+ ((KylinException) e).getErrorCode());
}
}
@@ -133,4 +148,156 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
String resSql = "select ab from table where aa = '' and bb = '\\'as\\'n\\''";
Assert.assertEquals(resSql, PushDownUtil.replaceEscapedQuote(sql));
}
+
+ @Test
+ public void testGenerateFlatTableSql() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1";
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, project, false));
+ }
+
+ @Test
+ public void testGenerateFlatTableSqlWithCCJoin() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change join condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ List<JoinTableDesc> joinTables = copyForWrite.getJoinTables();
+ TableRef rootTableRef = copyForWrite.getRootFactTable();
+ TblColRef cc1 = rootTableRef.getColumn("CC1");
+ JoinDesc join = joinTables.get(0).getJoin();
+ join.setForeignKeyColumns(new TblColRef[] { cc1 });
+ join.setForeignKey(new String[] { "TEST_BANK_INCOME.CC1" });
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n"
+ + "WHERE\n" //
+ + "1 = 1";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+
+ }
+
+ @Test
+ public void testGenerateFlatTableSqlWithFilterCondition() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change filter condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ copyForWrite.setFilterCondition(
+ "SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china'");
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1\n" //
+ + " AND (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4) = 'china' and (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)) = 'china')";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ }
+
+ @Test
+ public void testGenerateFlatTableSqlWithSpecialFunctions() {
+ String project = "default";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change filter condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ copyForWrite.setFilterCondition("timestampadd(day, 1, current_date) = '2012-01-01' and cc1 = 'china'");
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1\n" //
+ + " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)) = 'china')";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ }
+
+ private void updateModelToAddCC(String project, NDataModel model) {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ ComputedColumnDesc cc = new ComputedColumnDesc();
+ cc.setColumnName("CC1");
+ cc.setDatatype("int");
+ cc.setExpression("substring(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4)");
+ cc.setInnerExpression("SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)");
+ cc.setTableAlias("TEST_BANK_INCOME");
+ cc.setTableIdentity(model.getRootFactTableName());
+ copyForWrite.getComputedColumnDescs().add(cc);
+ List<NDataModel.NamedColumn> columns = copyForWrite.getAllNamedColumns();
+ int id = columns.size();
+ NDataModel.NamedColumn namedColumn = new NDataModel.NamedColumn();
+ namedColumn.setName("CC1");
+ namedColumn.setId(id);
+ namedColumn.setAliasDotColumn("TEST_BANK_INCOME.CC1");
+ columns.add(namedColumn);
+ copyForWrite.setAllNamedColumns(columns);
+ });
+ return null;
+ }, project);
+ }
}
[kylin] 17/38: KYLIN-5521 [FOLLOW UP] fix LeftOrInner Join & unstable UT
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2e60700e52213661efe22020ceb76b7557ffd635
Author: Jiale He <ji...@gmail.com>
AuthorDate: Fri Feb 24 11:51:13 2023 +0800
KYLIN-5521 [FOLLOW UP] fix LeftOrInner Join & unstable UT
---
.../main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
index b6342fd476..2029fd61ac 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
@@ -349,13 +349,14 @@ public class JoinsGraph implements Serializable {
for (Edge edge : edgeSet) {
if (!edge.isLeftJoin() || edge.isLeftOrInnerJoin()) {
TableRef fkSide = edge.fkSide();
- List<Edge> edgeList = inwardEdges(fkSide);
+ List<Edge> edgeList = inwardEdges(fkSide).stream().filter(e -> !e.isSwapJoin())
+ .collect(Collectors.toList());
if (CollectionUtils.isEmpty(edgeList)) {
continue;
}
for (Edge targetEdge : edgeList) {
if (!edge.equals(targetEdge) && fkSide.equals(targetEdge.pkSide())
- && !targetEdge.isLeftOrInnerJoin() && targetEdge.isLeftJoin()) {
+ && !targetEdge.isLeftOrInnerJoin()) {
setJoinToLeftOrInner(targetEdge.join);
normalize();
}
[kylin] 04/38: KYLIN-5523 computed column as join key & partition column
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 520e7c67eaf1c070ea0de1cddac99a5dc69263e7
Author: Pengfei Zhan <pe...@kyligence.io>
AuthorDate: Tue Feb 7 22:15:40 2023 +0800
KYLIN-5523 computed column as join key & partition column
---
.../org/apache/kylin/common/KylinConfigBase.java | 6 +-
.../org/apache/kylin/common/msg/CnMessage.java | 2 +-
.../java/org/apache/kylin/common/msg/Message.java | 2 +-
.../util/{StringUtil.java => StringHelper.java} | 85 ++-
.../common/util/NLocalFileMetadataTestCase.java | 2 +-
.../apache/kylin/common/util/StringHelperTest.java | 104 ++++
.../kylin/job/execution/AbstractExecutable.java | 16 +-
.../job/execution/DefaultExecutableOnModel.java | 8 +-
.../job/execution/EmailNotificationContent.java | 18 +-
.../apache/kylin/dimension/DimensionEncoding.java | 6 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 97 ++--
.../apache/kylin/metadata/model/JoinsGraph.java | 610 +++++++++++++++++++++
.../kylin/metadata/model/ModelDimensionDesc.java | 4 +-
.../kylin/metadata/model/MultiPartitionDesc.java | 2 +-
.../apache/kylin/metadata/model/NDataModel.java | 25 +-
.../apache/kylin/metadata/model/PartitionDesc.java | 10 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 32 +-
.../metadata/model/util/ComputedColumnUtil.java | 74 +--
.../kylin/metadata/project/ProjectInstance.java | 6 +-
.../kylin/source/adhocquery/IPushDownRunner.java | 18 +-
.../cube/planner/algorithm/AlgorithmTestBase.java | 6 +-
.../kylin/metadata/model/JoinsGraphTest.java | 4 +-
.../model/util/ComputedColumnUtilTest.java | 4 +-
.../kylin/rest/controller/SampleController.java | 6 +-
.../rest/controller/open/OpenSampleController.java | 7 +-
.../controller/open/OpenSampleControllerTest.java | 10 +-
.../org/apache/kylin/rest/service/JobService.java | 5 +-
.../apache/kylin/rest/service/SnapshotService.java | 86 +--
.../kylin/rest/service/ModelServiceBuildTest.java | 7 +-
.../sdk/datasource/PushDownRunnerSDKImpl.java | 4 +-
.../kylin/rest/service/SparkSourceService.java | 22 +-
.../apache/kylin/rest/service/TableService.java | 53 +-
.../kylin/metadata/model/AntiFlatCheckerTest.java | 4 +-
.../metadata/model/ColExcludedCheckerTest.java | 4 +-
.../kylin/newten/NBadQueryAndPushDownTest.java | 6 +-
.../java/org/apache/kylin/query/KylinTestBase.java | 16 +-
.../java/org/apache/kylin/util/ExecAndComp.java | 3 +-
.../kylin/rest/controller/NTableController.java | 20 +-
.../rest/controller/NTableControllerTest.java | 10 +-
.../kylin/rest/service/ModelSemanticHelper.java | 21 +-
.../apache/kylin/rest/service/ModelService.java | 57 +-
.../kylin/rest/service/ModelServiceTest.java | 57 +-
.../kylin/rest/service/TableServiceTest.java | 132 +++--
.../org/apache/kylin/query/util/PushDownUtil.java | 353 ++++++++----
.../apache/kylin/query/util/QueryAliasMatcher.java | 23 +-
.../org/apache/kylin/query/util/QueryUtil.java | 416 +++-----------
.../kylin/rest/service/ModelServiceQueryTest.java | 3 +-
src/query/pom.xml | 9 +-
.../optrule/AbstractAggCaseWhenFunctionRule.java | 18 +-
.../optrule/CountDistinctCaseWhenFunctionRule.java | 17 +-
.../query/optrule/KapAggFilterTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggJoinTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggProjectMergeRule.java | 4 +-
.../query/optrule/KapAggProjectTransposeRule.java | 4 +-
.../query/optrule/KapCountDistinctJoinRule.java | 4 +-
.../kap/query/optrule/KapSumCastTransposeRule.java | 19 +-
.../main/java/org/apache/kylin/query/QueryCli.java | 78 ---
.../kylin/query/engine/QueryRoutingEngine.java | 16 +-
.../org/apache/kylin/query/util/RuleUtils.java | 150 +++++
.../apache/kylin/query/util/PushDownUtilTest.java | 80 ++-
.../org/apache/kylin/query/util/QueryUtilTest.java | 336 +++++-------
.../kap/secondstorage/tdvt/TDVTHiveTest.java | 4 +-
.../service/ModelServiceWithSecondStorageTest.java | 3 +-
.../engine/spark/application/SparkApplication.java | 27 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 4 +-
.../engine/spark/utils/ComputedColumnEvalUtil.java | 52 +-
.../engine/spark/builder/CreateFlatTable.scala | 12 +-
.../engine/spark/builder/DFBuilderHelper.scala | 12 +-
.../engine/spark/builder/SegmentFlatTable.scala | 23 +-
.../kylin/engine/spark/job/FlatTableHelper.scala | 18 +-
.../job/stage/build/FlatTableAndDictBase.scala | 15 +-
.../spark/smarter/IndexDependencyParser.scala | 23 +-
.../query/pushdown/PushDownRunnerJdbcImpl.java | 4 +-
.../query/pushdown/PushDownRunnerSparkImpl.java | 3 +-
.../pushdown/PushDownRunnerSparkImplTest.java | 15 +-
.../kylin/engine/spark/job/NSparkCubingUtil.java | 11 -
.../scala/io/kyligence/kap/common/SSSource.scala | 89 +++
.../kylin/streaming/CreateStreamingFlatTable.scala | 24 +-
78 files changed, 2150 insertions(+), 1398 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f1770e9fb2..0d4796e6e3 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -72,7 +72,7 @@ import org.apache.kylin.common.util.FileUtils;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.SizeConvertUtil;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.common.util.Unsafe;
import org.slf4j.Logger;
@@ -2123,7 +2123,7 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.query.pushdown.partition-check.runner-class-name", "");
}
- public String getPartitionCheckRunnerClassNameWithDefaultValue() {
+ public String getDefaultPartitionCheckerClassName() {
String partitionCheckRunner = getPartitionCheckRunnerClassName();
if (StringUtils.isEmpty(partitionCheckRunner)) {
partitionCheckRunner = "org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl";
@@ -2790,7 +2790,7 @@ public abstract class KylinConfigBase implements Serializable {
public String getEngineWriteFs() {
String engineWriteFs = getOptional("kylin.env.engine-write-fs", "");
- return StringUtil.dropSuffix(engineWriteFs, File.separator);
+ return StringHelper.dropSuffix(engineWriteFs, File.separator);
}
public boolean isAllowedProjectAdminGrantAcl() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index 679e15176f..d36e2c2c41 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -997,7 +997,7 @@ public class CnMessage extends Message {
@Override
public String getBadSqlColumnNotFoundReason() {
- return "无法找到列 \"%s\"。请检查此列是否在源表中存在。若存在，可尝试重载表；若不存在，请联系管理员添加。";
+ return "无法找到列 \"%s\"。请检查此列是否在源表或可计算列中存在。若存在，可尝试重载表；若不存在，请联系管理员添加。";
}
@Override
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 32ef6010b5..ade226726c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -957,7 +957,7 @@ public class Message {
}
public String getBadSqlColumnNotFoundReason() {
- return "Can’t find column \"%s\". Please check if it exists in the source table. If exists, please try reloading the table; if not exist, please contact admin to add it.";
+ return "Can’t find column \"%s\". Please check if it exists in the source table or computed columns. If exists, please try reloading the table; if not exist, please contact admin to add it.";
}
public String getBadSqlColumnNotFoundSuggest() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java
similarity index 65%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
rename to src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java
index 5776a5895b..7be0bc9e07 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java
@@ -21,18 +21,21 @@ package org.apache.kylin.common.util;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.Locale;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-/**
- */
-public class StringUtil {
+public class StringHelper {
+
+ public static final char QUOTE = '\'';
+ public static final char DOUBLE_QUOTE = '"';
+ public static final char BACKTICK = '`';
- public static String[] EMPTY_ARRAY = new String[0];
+ private StringHelper() {
+ }
public static String[] filterSystemArgs(String[] args) {
List<String> whatsLeft = Lists.newArrayList();
@@ -86,7 +89,7 @@ public class StringUtil {
if (source != null) {
for (int i = 0; i < source.length; i++) {
if (source[i] != null) {
- target[i] = source[i].toUpperCase(Locale.ROOT);
+ target[i] = StringUtils.upperCase(source[i]);
}
}
}
@@ -175,7 +178,7 @@ public class StringUtil {
if (!s.isEmpty())
r.add(s);
}
- return r.toArray(new String[r.size()]);
+ return r.toArray(new String[0]);
}
// calculating length in UTF-8 of Java String without actually encoding it
@@ -213,4 +216,72 @@ public class StringUtil {
return str.split(splitBy);
}
+ public static String backtickToDoubleQuote(String expression) {
+ return convert(expression, StringHelper.BACKTICK, StringHelper.DOUBLE_QUOTE);
+ }
+
+ public static String doubleQuoteToBacktick(String expression) {
+ return convert(expression, StringHelper.DOUBLE_QUOTE, StringHelper.BACKTICK);
+ }
+
+ private static String convert(String expression, char srcQuote, char targetQuote) {
+ char[] chars = expression.toCharArray();
+ List<Integer> indexList = StringHelper.findQuoteIndexes(srcQuote, expression);
+ for (Integer integer : indexList) {
+ chars[integer] = targetQuote;
+ }
+ return new String(chars);
+ }
+
+ public static String backtickQuote(String identifier) {
+ String str = StringUtils.remove(identifier, StringHelper.BACKTICK);
+ return StringHelper.BACKTICK + str + StringHelper.BACKTICK;
+ }
+
+ public static String doubleQuote(String identifier) {
+ String str = StringUtils.remove(identifier, StringHelper.DOUBLE_QUOTE);
+ return StringHelper.DOUBLE_QUOTE + str + StringHelper.DOUBLE_QUOTE;
+ }
+
+ /**
+ * Search identifier quotes in the sql string.
+ * @param key the char to search
+ * @param str the input string
+ * @return index list of {@code key}
+ */
+ public static List<Integer> findQuoteIndexes(char key, String str) {
+ Preconditions.checkState(key == BACKTICK || key == DOUBLE_QUOTE);
+ char[] chars = str.toCharArray();
+ List<Integer> indexList = Lists.newArrayList();
+ List<Pair<Integer, Character>> toMatchTokens = Lists.newArrayList();
+ for (int i = 0; i < chars.length; i++) {
+ char ch = chars[i];
+ if (toMatchTokens.isEmpty()) {
+ if (ch == key || ch == QUOTE) {
+ toMatchTokens.add(new Pair<>(i, ch));
+ }
+ continue;
+ }
+
+ // The toMatchTokens is not empty, try to collect
+ Character ex = toMatchTokens.get(toMatchTokens.size() - 1).getSecond();
+ if (ch == ex && ch == key) {
+ toMatchTokens.add(new Pair<>(i, ex));
+ Preconditions.checkState(toMatchTokens.size() == 2);
+ indexList.add(toMatchTokens.get(0).getFirst());
+ indexList.add(toMatchTokens.get(1).getFirst());
+ toMatchTokens.clear();
+ } else if (ch == ex && ch == QUOTE) {
+ // There are two kind of single quote in the char array.
+ // One kind has two successive single quote '', we need to clear the toMatchTokens.
+ // Another kind has a form of \', just ignore it and go on match the next char.
+ Preconditions.checkState(toMatchTokens.size() == 1);
+ if (chars[i - 1] != '\\') {
+ toMatchTokens.clear();
+ }
+ }
+ }
+ Preconditions.checkState(indexList.size() % 2 == 0);
+ return indexList;
+ }
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java b/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java
index 212dea3c0c..bcab2b07d6 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java
@@ -147,7 +147,7 @@ public class NLocalFileMetadataTestCase extends AbstractTestCase {
val kylinHomePath = new File(getTestConfig().getMetadataUrl().toString()).getParentFile().getAbsolutePath();
overwriteSystemProp("KYLIN_HOME", kylinHomePath);
val jobJar = org.apache.kylin.common.util.FileUtils.findFile(
- new File(kylinHomePath, "../../../assembly/target/").getAbsolutePath(), "kap-assembly(.?)\\.jar");
+ new File(kylinHomePath, "../../../assembly/target/").getAbsolutePath(), "ke-assembly(.*?)\\.jar");
getTestConfig().setProperty("kylin.engine.spark.job-jar", jobJar == null ? "" : jobJar.getAbsolutePath());
getTestConfig().setProperty("kylin.query.security.acl-tcr-enabled", "false");
getTestConfig().setProperty("kylin.streaming.enabled", "true");
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/StringHelperTest.java b/src/core-common/src/test/java/org/apache/kylin/common/util/StringHelperTest.java
new file mode 100644
index 0000000000..dcadc2a013
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/util/StringHelperTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class StringHelperTest {
+
+ @Test
+ void testMin() {
+ Assertions.assertEquals("", StringHelper.min(null, ""));
+ Assertions.assertEquals("", StringHelper.min("", null));
+ Assertions.assertEquals("1", StringHelper.min("1", "2"));
+ }
+
+ @Test
+ void testMax() {
+ Assertions.assertEquals("", StringHelper.max(null, ""));
+ Assertions.assertEquals("", StringHelper.max("", null));
+ Assertions.assertEquals("2", StringHelper.max("1", "2"));
+ }
+
+ @Test
+ void testValidateBoolean() {
+ Assertions.assertTrue(StringHelper.validateBoolean("true"));
+ Assertions.assertTrue(StringHelper.validateBoolean("false"));
+ }
+
+ @Test
+ void testBacktickToDoubleQuote() {
+ Assertions.assertEquals("\"a\".\"b\" + 1", StringHelper.backtickToDoubleQuote("`a`.`b` + 1"));
+ }
+
+ @Test
+ void testDoubleQuoteToBackTick() {
+ Assertions.assertEquals("`a`.`b` + '''1'", StringHelper.doubleQuoteToBacktick("\"a\".\"b\" + '''1'"));
+ }
+
+ @Test
+ void testBacktickQuote() {
+ Assertions.assertEquals("`aa`", StringHelper.backtickQuote("aa"));
+ }
+
+ @Test
+ void testDoubleQuote() {
+ Assertions.assertEquals("\"aa\"", StringHelper.doubleQuote("aa"));
+ }
+
+ @Test
+ void testSubArray() {
+ String[] arr = { "a", "b", "c" };
+ try {
+ StringHelper.subArray(arr, -1, 1);
+ Assertions.fail();
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ }
+
+ try {
+ StringHelper.subArray(arr, 2, 1);
+ Assertions.fail();
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ }
+
+ try {
+ StringHelper.subArray(arr, 1, 5);
+ Assertions.fail();
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ }
+
+ String[] arrNew = StringHelper.subArray(arr, 0, 2);
+ Assertions.assertEquals(2, arrNew.length);
+ Assertions.assertEquals("a", arrNew[0]);
+ Assertions.assertEquals("b", arrNew[1]);
+ }
+
+ @Test
+ void testSplitAndTrim() {
+ String[] arr = StringHelper.splitAndTrim("a, ,b, c", ",");
+ Assertions.assertEquals(3, arr.length);
+ Assertions.assertEquals("a", arr[0]);
+ Assertions.assertEquals("b", arr[1]);
+ Assertions.assertEquals("c", arr[2]);
+ }
+}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 2805ee0b31..bfe6f1fb6a 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -50,7 +50,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.stream.Collectors;
-import io.kyligence.kap.guava20.shaded.common.base.Throwables;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
@@ -63,7 +62,7 @@ import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.MailHelper;
import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.common.util.ThrowableUtils;
import org.apache.kylin.job.constant.JobIssueEnum;
import org.apache.kylin.job.dao.ExecutableOutputPO;
@@ -82,13 +81,14 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import io.kyligence.kap.guava20.shaded.common.annotations.VisibleForTesting;
-import io.kyligence.kap.guava20.shaded.common.base.MoreObjects;
import lombok.Getter;
import lombok.Setter;
import lombok.val;
@@ -917,7 +917,7 @@ public abstract class AbstractExecutable implements Executable {
String layouts = getParam(NBatchConstants.P_LAYOUT_IDS);
if (layouts != null) {
- return computeDriverMemory(StringUtil.splitAndTrim(layouts, ",").length);
+ return computeDriverMemory(StringHelper.splitAndTrim(layouts, ",").length);
}
return 0;
}
@@ -925,8 +925,8 @@ public abstract class AbstractExecutable implements Executable {
public static Integer computeDriverMemory(Integer cuboidNum) {
KylinConfig config = KylinConfig.getInstanceFromEnv();
int[] driverMemoryStrategy = config.getSparkEngineDriverMemoryStrategy();
- List strategy = Lists.newArrayList(cuboidNum);
- Arrays.stream(driverMemoryStrategy).forEach(x -> strategy.add(Integer.valueOf(x)));
+ List<Integer> strategy = Lists.newArrayList(cuboidNum);
+ Arrays.stream(driverMemoryStrategy).forEach(strategy::add);
Collections.sort(strategy);
int index = strategy.indexOf(cuboidNum);
int driverMemoryMaximum = config.getSparkEngineDriverMemoryMaximum();
@@ -938,7 +938,7 @@ public abstract class AbstractExecutable implements Executable {
@Override
public String toString() {
- return MoreObjects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus())
+ return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus())
.toString();
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
index 339efeef8d..77d31bec84 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
@@ -23,12 +23,10 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
@@ -36,8 +34,10 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.cube.model.SegmentPartition;
import org.apache.kylin.metadata.model.ManagementType;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
import lombok.Getter;
import lombok.Setter;
@@ -108,7 +108,7 @@ public class DefaultExecutableOnModel extends DefaultExecutable {
val indexPlanManager = NIndexPlanManager.getInstance(getConfig(), getProject());
val indexPlan = indexPlanManager.getIndexPlan(getTargetModel());
val allLayoutIds = indexPlan.getAllLayouts().stream().map(l -> l.getId() + "").collect(Collectors.toSet());
- return Stream.of(StringUtil.splitAndTrim(layouts, ",")).anyMatch(allLayoutIds::contains);
+ return Stream.of(StringHelper.splitAndTrim(layouts, ",")).anyMatch(allLayoutIds::contains);
}
private boolean checkTargetSegmentAndPartitionExists(String segmentId) {
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/EmailNotificationContent.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/EmailNotificationContent.java
index 5106cc16cc..4d9074cdd3 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/EmailNotificationContent.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/EmailNotificationContent.java
@@ -24,7 +24,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.NonCustomProjectLevelConfig;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.constant.JobIssueEnum;
import org.apache.kylin.job.util.MailNotificationUtil;
@@ -89,7 +89,7 @@ public class EmailNotificationContent {
if (state == ExecutableState.ERROR) {
checkErrorTask(executable, dataMap, tasks);
dataMap.put("error_log",
- Matcher.quoteReplacement(StringUtil.noBlank(output.getShortErrMsg(), "no error message")));
+ Matcher.quoteReplacement(StringHelper.noBlank(output.getShortErrMsg(), "no error message")));
}
return Pair.newPair(getMailTitle(state, executable), getMailContent(state, dataMap));
@@ -116,7 +116,7 @@ public class EmailNotificationContent {
dataMap.put("error_step", errorTask.getName());
if (errorTask.getOutput().getExtra().containsKey(ExecutableConstants.MR_JOB_ID)) {
final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
- dataMap.put(ExecutableConstants.MR_JOB_ID, StringUtil.noBlank(mrJobId, "Not initialized"));
+ dataMap.put(ExecutableConstants.MR_JOB_ID, StringHelper.noBlank(mrJobId, "Not initialized"));
} else {
dataMap.put(ExecutableConstants.MR_JOB_ID, MailNotificationUtil.NA);
}
@@ -127,26 +127,26 @@ public class EmailNotificationContent {
AbstractExecutable executable) {
logger.info("notify on metadata persist exception: {}", exception.getMessage());
Map<String, Object> dataMap = getDataMap(executable);
- dataMap.put("error_log", Matcher.quoteReplacement(StringUtil.noBlank(
+ dataMap.put("error_log", Matcher.quoteReplacement(StringHelper.noBlank(
exception.getMessage(), "no error message")));
String content = MailNotificationUtil.getMailContent(MailNotificationUtil.METADATA_PERSIST_FAIL, dataMap);
String title = MailNotificationUtil.getMailTitle("METADATA_PERSIST",
"FAIL",
- StringUtil.noBlank(executable.getConfig().getDeployEnv(), MailNotificationUtil.NA),
+ StringHelper.noBlank(executable.getConfig().getDeployEnv(), MailNotificationUtil.NA),
executable.getProject(),
- StringUtil.noBlank(executable.getTargetSubjectAlias(), MailNotificationUtil.NA));
+ StringHelper.noBlank(executable.getTargetSubjectAlias(), MailNotificationUtil.NA));
return Pair.newPair(title, content);
}
private static Map<String, Object> getDataMap(AbstractExecutable executable) {
Map<String, Object> dataMap = Maps.newHashMap();
- dataMap.put("job_name", StringUtil.noBlank(executable.getName(), "missing job_name"));
+ dataMap.put("job_name", StringHelper.noBlank(executable.getName(), "missing job_name"));
dataMap.put("env_name", executable.getConfig().getDeployEnv());
- dataMap.put("submitter", StringUtil.noBlank(executable.getSubmitter(), "missing submitter"));
+ dataMap.put("submitter", StringHelper.noBlank(executable.getSubmitter(), "missing submitter"));
dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
dataMap.put("project_name", executable.getProject());
- dataMap.put("model_name", StringUtil.noBlank(executable.getTargetModelAlias(), "missing model_name"));
+ dataMap.put("model_name", StringHelper.noBlank(executable.getTargetModelAlias(), "missing model_name"));
return dataMap;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java b/src/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
index 1fa4e8e5e8..08465a7397 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dimension;
import java.io.Externalizable;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
/**
@@ -58,8 +58,8 @@ public abstract class DimensionEncoding implements Externalizable {
final String encodingName = parts[0];
final String[] encodingArgs = parts[parts.length - 1].isEmpty() //
- ? StringUtil.subArray(parts, 1, parts.length - 1)
- : StringUtil.subArray(parts, 1, parts.length);
+ ? StringHelper.subArray(parts, 1, parts.length - 1)
+ : StringHelper.subArray(parts, 1, parts.length);
return new Object[] { encodingName, encodingArgs };
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index e179e2bb63..49094bf6a3 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -19,9 +19,10 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import java.util.Locale;
import org.apache.calcite.avatica.util.Quoting;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -31,6 +32,9 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
import com.google.common.base.Preconditions;
+import lombok.Getter;
+import lombok.Setter;
+
/**
* Column Metadata from Source. All name should be uppercase.
* <p/>
@@ -40,27 +44,34 @@ public class ColumnDesc implements Serializable {
private static final String BACK_TICK = Quoting.BACK_TICK.string;
+ @Getter
@JsonProperty("id")
private String id;
@JsonProperty("name")
private String name;
+ @Getter
@JsonProperty("datatype")
private String datatype;
@JsonProperty("comment")
@JsonInclude(JsonInclude.Include.NON_NULL)
+ @Getter
+ @Setter
private String comment;
@JsonProperty("data_gen")
@JsonInclude(JsonInclude.Include.NON_NULL)
+ @Getter
private String dataGen;
@JsonProperty("index")
@JsonInclude(JsonInclude.Include.NON_NULL)
+ @Getter
private String index;
+ @Setter
@JsonProperty("cc_expr")
@JsonInclude(JsonInclude.Include.NON_NULL)
private String computedColumnExpr = null;//if null, it's not a computed column
@@ -73,11 +84,19 @@ public class ColumnDesc implements Serializable {
public boolean isPartitioned = false;
// parsed from data type
+ @Getter
private DataType type;
+ @Setter
private DataType upgradedType;
+ @Getter
+ @Setter
private TableDesc table;
+ @Getter
private int zeroBasedIndex = -1;
+
+ @Getter
+ @Setter
private boolean isNullable = true;
public ColumnDesc() { // default constructor for Jackson
@@ -107,27 +126,11 @@ public class ColumnDesc implements Serializable {
this.computedColumnExpr = computedColumnExpr;
}
- public void setComputedColumn(String exp) {
- computedColumnExpr = exp;
- }
-
- public int getZeroBasedIndex() {
- return zeroBasedIndex;
- }
-
- public String getDatatype() {
- return datatype;
- }
-
public void setDatatype(String datatype) {
this.datatype = datatype;
type = DataType.getType(datatype);
}
- public void setUpgradedType(DataType upgradedType) {
- this.upgradedType = upgradedType;
- }
-
public DataType getUpgradedType() {
if (this.upgradedType == null) {
return this.type;
@@ -136,10 +139,6 @@ public class ColumnDesc implements Serializable {
}
}
- public String getId() {
- return id;
- }
-
public void setId(String id) {
this.id = id;
if (id != null)
@@ -152,7 +151,7 @@ public class ColumnDesc implements Serializable {
}
public String getName() {
- return (name == null) ? null : name.toUpperCase(Locale.ROOT);
+ return StringUtils.upperCase(name);
}
public String getIdentity() {
@@ -177,26 +176,6 @@ public class ColumnDesc implements Serializable {
this.caseSensitiveName = caseSensitiveName;
}
- public TableDesc getTable() {
- return table;
- }
-
- public void setTable(TableDesc table) {
- this.table = table;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public DataType getType() {
- return type;
- }
-
public String getTypeName() {
return type.getName();
}
@@ -209,28 +188,20 @@ public class ColumnDesc implements Serializable {
return type.getScale();
}
- public boolean isNullable() {
- return this.isNullable;
- }
-
- public void setNullable(boolean nullable) {
- this.isNullable = nullable;
- }
-
- public String getDataGen() {
- return dataGen;
- }
-
- public String getIndex() {
- return index;
- }
-
public String getComputedColumnExpr() {
Preconditions.checkState(computedColumnExpr != null);
-
return computedColumnExpr;
}
+ public String getDoubleQuoteInnerExpr() {
+ Preconditions.checkState(computedColumnExpr != null);
+ int quoteCnt = StringUtils.countMatches(computedColumnExpr, StringHelper.QUOTE);
+ if (quoteCnt == 0 || quoteCnt == 1) {
+ return computedColumnExpr.replace(StringHelper.BACKTICK, StringHelper.DOUBLE_QUOTE);
+ }
+ return StringHelper.backtickToDoubleQuote(computedColumnExpr);
+ }
+
public boolean isComputedColumn() {
return computedColumnExpr != null;
}
@@ -306,8 +277,12 @@ public class ColumnDesc implements Serializable {
@Override
public String toString() {
- return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\''
- + ", comment='" + comment + '\'' + '}';
+ return "ColumnDesc{" //
+ + "id='" + id //
+ + "', name='" + name //
+ + "', datatype='" + datatype //
+ + "', comment='" + comment //
+ + "'}";
}
public ColumnDesc copy() {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
new file mode 100644
index 0000000000..acd810fbec
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
@@ -0,0 +1,610 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.metadata.model;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+
+public class JoinsGraph implements Serializable {
+
+ public class Edge implements Serializable {
+
+ @Getter
+ private final JoinDesc join;
+ private final ColumnDesc[] leftCols;
+ private final ColumnDesc[] rightCols;
+ private final NonEquiJoinCondition nonEquiJoinCondition;
+
+ private Edge(JoinDesc join) {
+ this.join = join;
+
+ leftCols = new ColumnDesc[join.getForeignKeyColumns().length];
+ int i = 0;
+ for (TblColRef colRef : join.getForeignKeyColumns()) {
+ leftCols[i++] = colRef.getColumnDesc();
+ }
+
+ rightCols = new ColumnDesc[join.getPrimaryKeyColumns().length];
+ i = 0;
+ for (TblColRef colRef : join.getPrimaryKeyColumns()) {
+ rightCols[i++] = colRef.getColumnDesc();
+ }
+
+ nonEquiJoinCondition = join.getNonEquiJoinCondition();
+ }
+
+ public boolean isJoinMatched(JoinDesc other) {
+ return join.equals(other);
+ }
+
+ public boolean isNonEquiJoin() {
+ return nonEquiJoinCondition != null;
+ }
+
+ public boolean isLeftJoin() {
+ return !join.isLeftOrInnerJoin() && join.isLeftJoin();
+ }
+
+ public boolean isLeftOrInnerJoin() {
+ return join.isLeftOrInnerJoin();
+ }
+
+ public boolean isInnerJoin() {
+ return !join.isLeftOrInnerJoin() && join.isInnerJoin();
+ }
+
+ private TableRef left() {
+ return join.getFKSide();
+ }
+
+ private TableRef right() {
+ return join.getPKSide();
+ }
+
+ private boolean isFkSide(TableRef tableRef) {
+ return join.getFKSide().equals(tableRef);
+ }
+
+ private boolean isPkSide(TableRef tableRef) {
+ return join.getPKSide().equals(tableRef);
+ }
+
+ private TableRef other(TableRef tableRef) {
+ if (left().equals(tableRef)) {
+ return right();
+ } else if (right().equals(tableRef)) {
+ return left();
+ }
+ throw new IllegalArgumentException("table " + tableRef + " is not on the edge " + this);
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+
+ if (this.getClass() != that.getClass())
+ return false;
+
+ return joinEdgeMatcher.matches(this, (Edge) that);
+ }
+
+ @Override
+ public int hashCode() {
+ if (this.isLeftJoin()) {
+ return Objects.hash(isLeftJoin(), leftCols, rightCols);
+ } else {
+ if (Arrays.hashCode(leftCols) < Arrays.hashCode(rightCols)) {
+ return Objects.hash(isLeftJoin(), leftCols, rightCols);
+ } else {
+ return Objects.hash(isLeftJoin(), rightCols, leftCols);
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("Edge: ");
+ sb.append(left())
+ .append(isLeftJoin() ? " LEFT JOIN "
+ : isLeftOrInnerJoin() ? " LEFT OR INNER JOIN " : " INNER JOIN ")
+ .append(right()).append(" ON ")
+ .append(Arrays.toString(Arrays.stream(leftCols).map(ColumnDesc::getName).toArray())).append(" = ")
+ .append(Arrays.toString(Arrays.stream(rightCols).map(ColumnDesc::getName).toArray()));
+ return sb.toString();
+ }
+ }
+
+ private Edge edgeOf(JoinDesc join) {
+ return new Edge(join);
+ }
+
+ private static final IJoinEdgeMatcher DEFAULT_JOIN_EDGE_MATCHER = new DefaultJoinEdgeMatcher();
+ @Setter
+ private IJoinEdgeMatcher joinEdgeMatcher = DEFAULT_JOIN_EDGE_MATCHER;
+
+ /**
+ * compare:
+ * 1. JoinType
+ * 2. Columns on both sides
+ */
+ public interface IJoinEdgeMatcher extends Serializable {
+ boolean matches(@NonNull Edge join1, @NonNull Edge join2);
+ }
+
+ public static class DefaultJoinEdgeMatcher implements IJoinEdgeMatcher {
+ @Override
+ public boolean matches(@NonNull Edge join1, @NonNull Edge join2) {
+ if (join1.isLeftJoin() != join2.isLeftJoin() && !join1.isLeftOrInnerJoin() && !join2.isLeftOrInnerJoin()) {
+ return false;
+ }
+
+ if (!Objects.equals(join1.nonEquiJoinCondition, join2.nonEquiJoinCondition)) {
+ return false;
+ }
+
+ if (join1.isLeftJoin()) {
+ return columnDescEquals(join1.leftCols, join2.leftCols)
+ && columnDescEquals(join1.rightCols, join2.rightCols);
+ } else {
+ return (columnDescEquals(join1.leftCols, join2.leftCols)
+ && columnDescEquals(join1.rightCols, join2.rightCols))
+ || (columnDescEquals(join1.leftCols, join2.rightCols)
+ && columnDescEquals(join1.rightCols, join2.leftCols));
+ }
+ }
+
+ private boolean columnDescEquals(ColumnDesc[] a, ColumnDesc[] b) {
+ if (a.length != b.length) {
+ return false;
+ }
+
+ List<ColumnDesc> oneList = Lists.newArrayList(a);
+ List<ColumnDesc> anotherList = Lists.newArrayList(b);
+
+ for (ColumnDesc obj : oneList) {
+ anotherList.removeIf(dual -> columnDescEquals(obj, dual));
+ }
+ return anotherList.isEmpty();
+ }
+
+ protected boolean columnDescEquals(ColumnDesc a, ColumnDesc b) {
+ return Objects.equals(a, b);
+ }
+ }
+
+ @Getter
+ private final TableRef center;
+ private final Map<String, TableRef> nodes = new HashMap<>();
+ private final Set<Edge> edges = new HashSet<>();
+ private final Map<TableRef, List<Edge>> edgesFromNode = new HashMap<>();
+ private final Map<TableRef, List<Edge>> edgesToNode = new HashMap<>();
+
+ /**
+ * For model there's always a center, if there's only one tableScan it's the center.
+ * Otherwise the center is not determined, it's a linked graph, hard to tell the center.
+ */
+ public JoinsGraph(TableRef root, List<JoinDesc> joins) {
+ this.center = root;
+ addNode(root);
+
+ for (JoinDesc join : joins) {
+ Preconditions.checkState(Arrays.stream(join.getForeignKeyColumns()).allMatch(TblColRef::isQualified));
+ Preconditions.checkState(Arrays.stream(join.getPrimaryKeyColumns()).allMatch(TblColRef::isQualified));
+ addAsEdge(join);
+ }
+
+ validate(joins);
+ }
+
+ private void addNode(TableRef table) {
+ Preconditions.checkNotNull(table);
+ String alias = table.getAlias();
+ TableRef node = nodes.get(alias);
+ if (node != null) {
+ Preconditions.checkArgument(node.equals(table), "[%s]'s Alias \"%s\" has conflict with [%s].", table, alias,
+ node);
+ } else {
+ nodes.put(alias, table);
+ }
+ }
+
+ private void addAsEdge(JoinDesc join) {
+ TableRef fkTable = join.getFKSide();
+ TableRef pkTable = join.getPKSide();
+ addNode(pkTable);
+
+ Edge edge = edgeOf(join);
+ edgesFromNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
+ edgesFromNode.get(fkTable).add(edge);
+ edgesToNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
+ edgesToNode.get(pkTable).add(edge);
+ if (!edge.isLeftJoin()) {
+ // inner join is reversible
+ edgesFromNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
+ edgesFromNode.get(pkTable).add(edge);
+ edgesToNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
+ edgesToNode.get(fkTable).add(edge);
+ }
+ edges.add(edge);
+ }
+
+ public void setJoinToLeftOrInner(JoinDesc join) {
+ if (!join.isLeftJoin()) {
+ join.setLeftOrInner(true);
+ return;
+ }
+
+ join.setLeftOrInner(true);
+ TableRef fkTable = join.getFKSide();
+ TableRef pkTable = join.getPKSide();
+ Edge edge = edges.stream().filter(e -> e.isJoinMatched(join)).findFirst().orElse(null);
+ if (edge == null) {
+ return;
+ }
+ edgesFromNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
+ edgesFromNode.get(pkTable).add(edge);
+ edgesToNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
+ edgesToNode.get(fkTable).add(edge);
+ }
+
+ private void validate(List<JoinDesc> joins) {
+ for (JoinDesc join : joins) {
+ TableRef fkTable = join.getFKSide();
+ Preconditions.checkNotNull(nodes.get(fkTable.getAlias()));
+ Preconditions.checkState(nodes.get(fkTable.getAlias()).equals(fkTable));
+ }
+ Preconditions.checkState(nodes.size() == joins.size() + 1);
+ }
+
+ public boolean match(JoinsGraph pattern, Map<String, String> matchAlias) {
+ return match(pattern, matchAlias, false);
+ }
+
+ public boolean match(JoinsGraph pattern, Map<String, String> matchAlias, boolean matchPatial) {
+ return match(pattern, matchAlias, matchPatial, false);
+ }
+
+ public boolean match(JoinsGraph pattern, Map<String, String> matchAlias, boolean matchPatial,
+ boolean matchPartialNonEquiJoin) {
+ if (pattern == null || pattern.center == null) {
+ throw new IllegalArgumentException("pattern(model) should have a center: " + pattern);
+ }
+
+ List<TableRef> candidatesOfQCenter = searchCenterByIdentity(pattern.center);
+ if (CollectionUtils.isEmpty(candidatesOfQCenter)) {
+ return false;
+ }
+
+ for (TableRef queryCenter : candidatesOfQCenter) {
+ // query <-> pattern
+ Map<TableRef, TableRef> trialMatch = Maps.newHashMap();
+ trialMatch.put(queryCenter, pattern.center);
+
+ if (!checkInnerJoinNum(pattern, queryCenter, pattern.center, matchPatial)) {
+ continue;
+ }
+
+ AtomicReference<Map<TableRef, TableRef>> finalMatchRef = new AtomicReference<>();
+ innerMatch(pattern, trialMatch, matchPatial, finalMatchRef);
+ if (finalMatchRef.get() != null
+ && (matchPartialNonEquiJoin || checkNonEquiJoinMatches(finalMatchRef.get(), pattern))) {
+ matchAlias.clear();
+ matchAlias.putAll(finalMatchRef.get().entrySet().stream()
+ .collect(Collectors.toMap(e -> e.getKey().getAlias(), e -> e.getValue().getAlias())));
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static JoinsGraph normalizeJoinGraph(JoinsGraph joinsGraph) {
+ for (Edge edge : joinsGraph.edges) {
+ if (!edge.isLeftJoin() || edge.isLeftOrInnerJoin()) {
+ TableRef leftTable = edge.left();
+ List<Edge> edgeList = joinsGraph.edgesToNode.get(leftTable);
+ if (CollectionUtils.isEmpty(edgeList)) {
+ continue;
+ }
+ for (Edge targetEdge : edgeList) {
+ if (!edge.equals(targetEdge) && leftTable.equals(targetEdge.right())
+ && !targetEdge.isLeftOrInnerJoin()) {
+ joinsGraph.setJoinToLeftOrInner(targetEdge.join);
+ normalizeJoinGraph(joinsGraph);
+ }
+ }
+ }
+ }
+ return joinsGraph;
+ }
+
+ public List<TableRef> getAllTblRefNodes() {
+ return Lists.newArrayList(nodes.values());
+ }
+
+ /**
+ * check if any non-equi join is missed in the pattern
+ * if so, we cannot match the current graph with the the pattern graph.
+ * set `kylin.query.match-partial-non-equi-join-model` to skip this checking
+ * @param matches
+ * @return
+ */
+ private boolean checkNonEquiJoinMatches(Map<TableRef, TableRef> matches, JoinsGraph pattern) {
+ HashSet<TableRef> patternGraphTables = new HashSet<>(pattern.nodes.values());
+
+ for (TableRef patternTable : patternGraphTables) {
+ List<Edge> outgoingEdges = pattern.getEdgesByFKSide(patternTable);
+ // for all outgoing non-equi join edges
+ // if there is no match found for the right side table in the current graph
+ // return false
+ for (Edge outgoingEdge : outgoingEdges) {
+ if (outgoingEdge.isNonEquiJoin()) {
+ if (!matches.containsValue(patternTable) || !matches.containsValue(outgoingEdge.right())) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ private boolean isAllJoinInner(JoinsGraph joinsGraph, TableRef tableRef) {
+ List<Edge> edgesFromNode = joinsGraph.edgesFromNode.get(tableRef);
+ List<Edge> edgesToNode = joinsGraph.edgesToNode.get(tableRef);
+
+ if (edgesFromNode == null) {
+ return false;
+ }
+
+ if (edgesToNode == null) {
+ return false;
+ }
+
+ if (edgesToNode.size() != edgesFromNode.size()) {
+ return false;
+ }
+
+ for (Edge edge : edgesFromNode) {
+ if (edge.join.isLeftJoin()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private boolean checkInnerJoinNum(JoinsGraph pattern, TableRef queryTableRef, TableRef patternTableRef,
+ boolean matchPartial) {
+ if (matchPartial) {
+ return true;
+ }
+ // fully match: unmatched if extra inner join edge on either graph
+ // matched if: query graph join count: model graph join count:
+ // 1) inner join <= inner join
+ // 2) inner join + left or inner join >= inner join
+ List<Edge> innerQueryEdges = this.edgesFrom(queryTableRef).stream().filter(Edge::isInnerJoin)
+ .collect(Collectors.toList());
+ List<Edge> notLeftQueryEdges = this.edgesFrom(queryTableRef).stream().filter(e -> !e.isLeftJoin())
+ .collect(Collectors.toList());
+ List<Edge> innerPatternEdges = pattern.edgesFrom(patternTableRef).stream().filter(Edge::isInnerJoin)
+ .collect(Collectors.toList());
+
+ // if all joins are inner joins, compare sum of both sides
+ if (isAllJoinInner(this, queryTableRef) && isAllJoinInner(pattern, patternTableRef)) {
+ int cntInnerQueryEdges = innerQueryEdges.size();
+ int cntNotLeftQueryEdges = notLeftQueryEdges.size();
+ int cntInnerPatternEdges = innerPatternEdges.size();
+ return cntInnerQueryEdges <= cntInnerPatternEdges && cntNotLeftQueryEdges >= cntInnerPatternEdges;
+ }
+
+ // if not all joins are inner, compare left side and right side separately
+ // Calculate join count in query graph
+ int cntLeftSideInnerQueryEdges = (int) innerQueryEdges.stream()
+ .filter(edge -> edge.right().equals(queryTableRef)).count();
+ int cntRightSideInnerQueryEdges = (int) innerQueryEdges.stream()
+ .filter(edge -> edge.left().equals(queryTableRef)).count();
+ int cntLeftSideNotLeftQueryEdges = (int) notLeftQueryEdges.stream()
+ .filter(edge -> edge.right().equals(queryTableRef)).count();
+ int cntRightSideNotLeftQueryEdges = (int) notLeftQueryEdges.stream()
+ .filter(edge -> edge.left().equals(queryTableRef)).count();
+ // Calculate join count in model graph
+ int cntLeftSideInnerPatternEdges = (int) innerPatternEdges.stream()
+ .filter(edge -> edge.right().equals(patternTableRef)).count();
+ int cntRightSideInnerPatternEdges = (int) innerPatternEdges.stream()
+ .filter(edge -> edge.left().equals(patternTableRef)).count();
+
+ boolean isLeftEqual = cntLeftSideInnerQueryEdges <= cntLeftSideInnerPatternEdges
+ && cntLeftSideNotLeftQueryEdges >= cntLeftSideInnerPatternEdges;
+ boolean isRightEqual = cntRightSideInnerQueryEdges <= cntRightSideInnerPatternEdges
+ && cntRightSideNotLeftQueryEdges >= cntRightSideInnerPatternEdges;
+ return isLeftEqual && isRightEqual;
+ }
+
+ private void innerMatch(JoinsGraph pattern, Map<TableRef, TableRef> trialMatches, boolean matchPartial,
+ AtomicReference<Map<TableRef, TableRef>> finalMatch) {
+ if (trialMatches.size() == nodes.size()) {
+ //match is found
+ finalMatch.set(trialMatches);
+ return;
+ }
+
+ Preconditions.checkState(nodes.size() > trialMatches.size());
+ Optional<Pair<Edge, TableRef>> toMatch = trialMatches.keySet().stream()
+ .map(t -> edgesFrom(t).stream().filter(e -> !trialMatches.containsKey(e.other(t))).findFirst()
+ .map(edge -> new Pair<>(edge, edge.other(t))).orElse(null))
+ .filter(Objects::nonNull).findFirst();
+
+ Preconditions.checkState(toMatch.isPresent());
+ Edge toMatchQueryEdge = toMatch.get().getFirst();
+ TableRef toMatchQueryNode = toMatch.get().getSecond();
+ TableRef matchedQueryNode = toMatchQueryEdge.other(toMatchQueryNode);
+ TableRef matchedPatternNode = trialMatches.get(matchedQueryNode);
+
+ List<TableRef> toMatchPatternNodeCandidates = Lists.newArrayList();
+ for (Edge patternEdge : pattern.edgesFrom(matchedPatternNode)) {
+ TableRef toMatchPatternNode = patternEdge.other(matchedPatternNode);
+ if (!toMatchQueryNode.getTableIdentity().equals(toMatchPatternNode.getTableIdentity())
+ || !toMatchQueryEdge.equals(patternEdge) || trialMatches.containsValue(toMatchPatternNode)
+ || !checkInnerJoinNum(pattern, toMatchQueryNode, toMatchPatternNode, matchPartial)) {
+ continue;
+ }
+ toMatchPatternNodeCandidates.add(toMatchPatternNode);
+ }
+
+ for (TableRef toMatchPatternNode : toMatchPatternNodeCandidates) {
+ Map<TableRef, TableRef> newTrialMatches = Maps.newHashMap();
+ newTrialMatches.putAll(trialMatches);
+ newTrialMatches.put(toMatchQueryNode, toMatchPatternNode);
+ innerMatch(pattern, newTrialMatches, matchPartial, finalMatch);
+ if (finalMatch.get() != null) {
+ //get out of recursive invoke chain straightly
+ return;
+ }
+ }
+ }
+
+ public List<Edge> unmatched(JoinsGraph pattern) {
+ List<Edge> unmatched = Lists.newArrayList();
+ Set<Edge> all = edgesFromNode.values().stream().flatMap(List::stream).collect(Collectors.toSet());
+ for (Edge edge : all) {
+ List<JoinDesc> joins = getJoinsPathByPKSide(edge.right());
+ JoinsGraph subGraph = new JoinsGraph(center, joins);
+ if (subGraph.match(pattern, Maps.newHashMap())) {
+ continue;
+ }
+ unmatched.add(edge);
+ }
+ return unmatched;
+ }
+
+ private List<TableRef> searchCenterByIdentity(final TableRef table) {
+ // special case: several same nodes in a JoinGraph
+ return nodes.values().stream().filter(node -> node.getTableIdentity().equals(table.getTableIdentity()))
+ .filter(node -> {
+ List<JoinDesc> path2Center = getJoinsPathByPKSide(node);
+ return path2Center.stream().noneMatch(JoinDesc::isLeftJoin);
+ }).collect(Collectors.toList());
+ }
+
+ private List<Edge> edgesFrom(TableRef thisSide) {
+ return edgesFromNode.getOrDefault(thisSide, Lists.newArrayList());
+ }
+
+ public Map<String, String> matchAlias(JoinsGraph joinsGraph, KylinConfig kylinConfig) {
+ Map<String, String> matchAlias = Maps.newHashMap();
+ match(joinsGraph, matchAlias, kylinConfig.isQueryMatchPartialInnerJoinModel(),
+ kylinConfig.partialMatchNonEquiJoins());
+ return matchAlias;
+ }
+
+ public Map<String, String> matchAlias(JoinsGraph joinsGraph, boolean matchPartial) {
+ Map<String, String> matchAlias = Maps.newHashMap();
+ match(joinsGraph, matchAlias, matchPartial);
+ return matchAlias;
+ }
+
+ public List<Edge> getEdgesByFKSide(TableRef table) {
+ if (!edgesFromNode.containsKey(table)) {
+ return Lists.newArrayList();
+ }
+ return edgesFromNode.get(table).stream().filter(e -> e.isFkSide(table)).collect(Collectors.toList());
+ }
+
+ private Edge getEdgeByPKSide(TableRef table) {
+ if (!edgesToNode.containsKey(table)) {
+ return null;
+ }
+ List<Edge> edgesByPkSide = edgesToNode.get(table).stream().filter(e -> e.isPkSide(table))
+ .collect(Collectors.toList());
+ if (edgesByPkSide.isEmpty()) {
+ return null;
+ }
+ Preconditions.checkState(edgesByPkSide.size() == 1, "%s is allowed to be Join PK side once", table);
+ return edgesByPkSide.get(0);
+ }
+
+ public JoinDesc getJoinByPKSide(TableRef table) {
+ Edge edge = getEdgeByPKSide(table);
+ return edge != null ? edge.join : null;
+ }
+
+ private List<JoinDesc> getJoinsPathByPKSide(TableRef table) {
+ List<JoinDesc> pathToRoot = Lists.newArrayList();
+ TableRef pkSide = table; // start from leaf
+ while (pkSide != null) {
+ JoinDesc subJoin = getJoinByPKSide(pkSide);
+ if (subJoin != null) {
+ pathToRoot.add(subJoin);
+ pkSide = subJoin.getFKSide();
+ } else {
+ pkSide = null;
+ }
+ }
+ return Lists.reverse(pathToRoot);
+ }
+
+ public JoinsGraph getSubgraphByAlias(Set<String> aliasSets) {
+ TableRef subGraphRoot = this.center;
+ Set<JoinDesc> subGraphJoin = Sets.newHashSet();
+ for (String alias : aliasSets) {
+ subGraphJoin.addAll(getJoinsPathByPKSide(nodes.get(alias)));
+ }
+ return new JoinsGraph(subGraphRoot, Lists.newArrayList(subGraphJoin));
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder graphStrBuilder = new StringBuilder();
+ graphStrBuilder.append("Root: ").append(center);
+ List<Edge> nextEdges = getEdgesByFKSide(center);
+ nextEdges.forEach(e -> buildGraphStr(graphStrBuilder, e, 1));
+ return graphStrBuilder.toString();
+ }
+
+ private void buildGraphStr(StringBuilder sb, @NonNull Edge edge, int indent) {
+ sb.append('\n');
+ for (int i = 0; i < indent; i++) {
+ sb.append(" ");
+ }
+ sb.append(edge);
+ List<Edge> nextEdges = getEdgesByFKSide(edge.right());
+ nextEdges.forEach(e -> buildGraphStr(sb, e, indent + 1));
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index dd3ff1e6c5..cc7f6f4a32 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.List;
import java.util.Locale;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -60,7 +60,7 @@ public class ModelDimensionDesc implements Serializable {
public void init(NDataModel model) {
table = table.toUpperCase(Locale.ROOT);
if (columns != null) {
- StringUtil.toUpperCaseArray(columns, columns);
+ StringHelper.toUpperCaseArray(columns, columns);
}
if (model != null) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MultiPartitionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MultiPartitionDesc.java
index cf8c3916fa..859a2bc230 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MultiPartitionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MultiPartitionDesc.java
@@ -192,7 +192,7 @@ public class MultiPartitionDesc implements Serializable {
List<String> conditions = Lists.newArrayList();
for (int i = 0; i < columnRefs.size(); i++) {
final int x = i;
- String item = columnRefs.get(x).getBackTickExpressionInSourceDB() + " in (" + //
+ String item = columnRefs.get(x).getBackTickExp() + " in (" + //
values.stream().map(a -> generateFormattedValue(columnRefs.get(x).getType(), a[x]))
.collect(Collectors.joining(", "))
+ ")";
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 3d7cffbd17..c51b0cd5ff 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -53,7 +53,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.scheduler.SchedulerEventNotifier;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.model.tool.CalciteParser;
@@ -588,7 +588,7 @@ public class NDataModel extends RootPersistentEntity {
for (TableRef t : allTableRefs) {
if (t.getTableIdentity().equals(table.getIdentity())
- && StringUtil.equals(t.getTableDesc().getProject(), table.getProject()))
+ && StringUtils.equals(t.getTableDesc().getProject(), table.getProject()))
return true;
}
return false;
@@ -822,23 +822,17 @@ public class NDataModel extends RootPersistentEntity {
continue;
}
} else {
- if (0 == quotationType) {
- quotationType = 1;
- continue;
- }
+ quotationType = 1;
+ continue;
}
}
if ('"' == this.filterCondition.charAt(i)) {
if (quotationType > 0) {
if (2 == quotationType) {
quotationType = 0;
- continue;
}
} else {
- if (0 == quotationType) {
- quotationType = 2;
- continue;
- }
+ quotationType = 2;
}
}
}
@@ -849,11 +843,12 @@ public class NDataModel extends RootPersistentEntity {
for (JoinTableDesc joinTable : joinTables) {
TableRef dimTable = joinTable.getTableRef();
JoinDesc join = joinTable.getJoin();
- if (join == null)
+ if (join == null) {
throw new IllegalStateException("Missing join conditions on table " + dimTable);
+ }
- StringUtil.toUpperCaseArray(join.getForeignKey(), join.getForeignKey());
- StringUtil.toUpperCaseArray(join.getPrimaryKey(), join.getPrimaryKey());
+ StringHelper.toUpperCaseArray(join.getForeignKey(), join.getForeignKey());
+ StringHelper.toUpperCaseArray(join.getPrimaryKey(), join.getPrimaryKey());
// primary key
String[] pks = join.getPrimaryKey();
@@ -1082,7 +1077,6 @@ public class NDataModel extends RootPersistentEntity {
}
private ImmutableBiMap<Integer, TblColRef> initAllNamedColumns(Predicate<NamedColumn> filter) {
- List<TblColRef> all = new ArrayList<>(allNamedColumns.size());
ImmutableBiMap.Builder<Integer, TblColRef> mapBuilder = ImmutableBiMap.builder();
for (NamedColumn d : allNamedColumns) {
if (!d.isExist()) {
@@ -1090,7 +1084,6 @@ public class NDataModel extends RootPersistentEntity {
}
TblColRef col = this.findColumn(d.aliasDotColumn);
d.aliasDotColumn = col.getIdentity();
- all.add(col);
if (filter.test(d)) {
mapBuilder.put(d.id, col);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 47b0adc805..d7717b1771 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -307,7 +307,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondAsDate(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive, String partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getBackTickExpressionInSourceDB();
+ String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ").append(String.format(Locale.ROOT, "to_date('%s', '%s')",
DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat), partitionColumnDateFormat));
builder.append(and);
@@ -317,7 +317,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondAsTimestamp(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getBackTickExpressionInSourceDB();
+ String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ").append(startInclusive);
builder.append(and);
builder.append(partitionColumnName).append(" < ").append(endExclusive);
@@ -325,7 +325,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondAsYmInt(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getBackTickExpressionInSourceDB();
+ String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ")
.append(DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_MONTH_PATTERN));
builder.append(and);
@@ -335,7 +335,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getBackTickExpressionInSourceDB();
+ String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ")
.append(DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN));
builder.append(and);
@@ -345,7 +345,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive, String partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getBackTickExpressionInSourceDB();
+ String partitionColumnName = partitionColumn.getBackTickExp();
if (endExclusive <= startInclusive) {
builder.append("1=1");
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index b6cd7ebd2a..1c2b132897 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -18,12 +18,9 @@
package org.apache.kylin.metadata.model;
-import static com.google.common.base.Preconditions.checkArgument;
-
import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
-import java.util.Locale;
import java.util.Set;
import org.apache.calcite.avatica.util.Quoting;
@@ -32,12 +29,13 @@ import org.apache.calcite.sql.SqlOperator;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
+import com.google.common.base.Preconditions;
+
import lombok.Getter;
import lombok.Setter;
/**
*/
-@SuppressWarnings({ "serial" })
public class TblColRef implements Serializable {
private static final String INNER_TABLE_NAME = "_kylin_table";
@@ -128,8 +126,7 @@ public class TblColRef implements Serializable {
column.setDatatype(DYNAMIC_DATA_TYPE);
TableDesc table = new TableDesc();
column.setTable(table);
- TblColRef colRef = new TblColRef(column);
- return colRef;
+ return new TblColRef(column);
}
private static final NDataModel UNKNOWN_MODEL = new NDataModel();
@@ -142,14 +139,14 @@ public class TblColRef implements Serializable {
}
public static TblColRef columnForUnknownModel(TableRef table, ColumnDesc colDesc) {
- checkArgument(table.getModel() == UNKNOWN_MODEL);
+ Preconditions.checkArgument(table.getModel() == UNKNOWN_MODEL);
return new TblColRef(table, colDesc);
}
public static void fixUnknownModel(NDataModel model, String alias, TblColRef col) {
- checkArgument(col.table.getModel() == UNKNOWN_MODEL || col.table.getModel() == model);
+ Preconditions.checkArgument(col.table.getModel() == UNKNOWN_MODEL || col.table.getModel() == model);
TableRef tableRef = model.findTable(alias);
- checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity()));
+ Preconditions.checkArgument(tableRef.getTableDesc().getIdentity().equals(col.column.getTable().getIdentity()));
col.fixTableRef(tableRef);
}
@@ -199,7 +196,7 @@ public class TblColRef implements Serializable {
}
public TblColRef(TableRef table, ColumnDesc column) {
- checkArgument(table.getTableDesc().getIdentity().equals(column.getTable().getIdentity()));
+ Preconditions.checkArgument(table.getTableDesc().getIdentity().equals(column.getTable().getIdentity()));
this.table = table;
this.column = column;
}
@@ -251,18 +248,15 @@ public class TblColRef implements Serializable {
}
}
- public String getDoubleQuoteExpressionInSourceDB() {
+ public String getDoubleQuoteExp() {
if (column.isComputedColumn())
- return column.getComputedColumnExpr();
+ return column.getDoubleQuoteInnerExpr();
return wrapIdentity(DOUBLE_QUOTE);
}
- public String getBackTickExpressionInSourceDB() {
- if (column.isComputedColumn())
- return column.getComputedColumnExpr();
-
- return wrapIdentity(BACK_TICK);
+ public String getBackTickExp() {
+ return column.isComputedColumn() ? column.getComputedColumnExpr() : wrapIdentity(BACK_TICK);
}
public String getTable() {
@@ -377,13 +371,13 @@ public class TblColRef implements Serializable {
if (column.getTable() == null) {
return "NULL";
} else {
- return column.getTable().getIdentity().toUpperCase(Locale.ROOT);
+ return StringUtils.upperCase(column.getTable().getIdentity());
}
}
// return DB.TABLE.COLUMN
public String getColumnWithTableAndSchema() {
- return (getTableWithSchema() + "." + column.getName()).toUpperCase(Locale.ROOT);
+ return StringUtils.upperCase(getTableWithSchema() + "." + column.getName());
}
public boolean isCastInnerColumn() {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
index 773afd993d..09435140f8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.BadModelException;
import org.apache.kylin.metadata.model.BadModelException.CauseType;
@@ -89,7 +88,7 @@ public class ComputedColumnUtil {
JoinsGraph newCCGraph = getCCExprRelatedSubgraph(newCC, newModel);
for (NDataModel existingModel : otherModels) {
for (ComputedColumnDesc existingCC : existingModel.getComputedColumnDescs()) {
- if (!StringUtil.equals(newCC.getTableIdentity(), existingCC.getTableIdentity())) {
+ if (!StringUtils.equals(newCC.getTableIdentity(), existingCC.getTableIdentity())) {
continue;
}
JoinsGraph existCCGraph = getCCExprRelatedSubgraph(existingCC, existingModel);
@@ -107,16 +106,6 @@ public class ComputedColumnUtil {
return null;
}
- public static BiMap<String, String> getAllCCNameAndExp(List<NDataModel> allModels) {
- BiMap<String, String> allCCNameAndExp = HashBiMap.create();
- for (NDataModel otherModel : allModels) {
- for (ComputedColumnDesc cc : otherModel.getComputedColumnDescs()) {
- allCCNameAndExp.put(cc.getColumnName(), cc.getExpression());
- }
- }
- return allCCNameAndExp;
- }
-
public static class ExprIdentifierFinder extends SqlBasicVisitor<SqlNode> {
List<Pair<String, String>> columnWithTableAlias;
@@ -147,7 +136,6 @@ public class ComputedColumnUtil {
@Override
public SqlNode visit(SqlIdentifier id) {
- //Preconditions.checkState(id.names.size() == 2, "error when get identifier in cc's expr");
if (id.names.size() == 2) {
columnWithTableAlias.add(Pair.newPair(id.names.get(0), id.names.get(1)));
}
@@ -155,10 +143,6 @@ public class ComputedColumnUtil {
}
}
- public static Map<String, Set<String>> getCCUsedColsMapWithProject(String project, ColumnDesc columnDesc) {
- return getCCUsedColsMapWithModel(getModel(project, columnDesc.getName()), columnDesc);
- }
-
public static Set<String> getCCUsedColsWithProject(String project, ColumnDesc columnDesc) {
NDataModel model = getModel(project, columnDesc.getName());
return getCCUsedColsWithModel(model, columnDesc);
@@ -201,21 +185,20 @@ public class ComputedColumnUtil {
public static Map<String, Set<String>> getCCUsedColsMap(NDataModel model, String colName) {
Map<String, Set<String>> usedCols = Maps.newHashMap();
Map<String, String> aliasTableMap = getAliasTableMap(model);
- Preconditions.checkState(aliasTableMap.size() > 0, "can not find cc:" + colName + "'s table alias");
+ Preconditions.checkState(aliasTableMap.size() > 0, "can not find cc:%s's table alias", colName);
ComputedColumnDesc targetCC = model.getComputedColumnDescs().stream()
.filter(cc -> cc.getColumnName().equalsIgnoreCase(colName)) //
.findFirst().orElse(null);
if (targetCC == null) {
- throw new RuntimeException("ComputedColumn(name: " + colName + ") is not on model: " + model.getUuid());
+ throw new IllegalStateException(
+ "ComputedColumn(name: " + colName + ") is not on model: " + model.getUuid());
}
List<Pair<String, String>> colsWithAlias = ExprIdentifierFinder.getExprIdentifiers(targetCC.getExpression());
for (Pair<String, String> cols : colsWithAlias) {
String tableIdentifier = aliasTableMap.get(cols.getFirst());
- if (!usedCols.containsKey(tableIdentifier)) {
- usedCols.put(tableIdentifier, Sets.newHashSet());
- }
+ usedCols.putIfAbsent(tableIdentifier, Sets.newHashSet());
usedCols.get(tableIdentifier).add(cols.getSecond());
}
return usedCols;
@@ -224,13 +207,12 @@ public class ComputedColumnUtil {
private static Set<String> getCCUsedCols(NDataModel model, String colName, String ccExpr) {
Set<String> usedCols = new HashSet<>();
Map<String, String> aliasTableMap = getAliasTableMap(model);
- Preconditions.checkState(aliasTableMap.size() > 0, "can not find cc:" + colName + "'s table alias");
+ Preconditions.checkState(aliasTableMap.size() > 0, "can not find cc:%s's table alias", colName);
List<Pair<String, String>> colsWithAlias = ExprIdentifierFinder.getExprIdentifiers(ccExpr);
for (Pair<String, String> cols : colsWithAlias) {
String tableIdentifier = aliasTableMap.get(cols.getFirst());
usedCols.add(tableIdentifier + "." + cols.getSecond());
}
- //Preconditions.checkState(usedCols.size() > 0, "can not find cc:" + columnDesc.getUuid() + "'s used cols");
return usedCols;
}
@@ -246,8 +228,7 @@ public class ComputedColumnUtil {
private static NDataModel getModel(String project, String ccName) {
List<NDataModel> models = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
.listUnderliningDataModels();
- for (NDataModel modelDesc : models) {
- NDataModel model = modelDesc;
+ for (NDataModel model : models) {
Set<String> computedColumnNames = model.getComputedColumnNames();
if (computedColumnNames.contains(ccName)) {
return model;
@@ -337,25 +318,6 @@ public class ComputedColumnUtil {
col2.getTableIdentity() + "." + col2.getColumnName());
}
- public static boolean isLiteralSameCCExpr(ComputedColumnDesc existingCC, ComputedColumnDesc newCC) {
- String definition0 = existingCC.getExpression();
- String definition1 = newCC.getExpression();
-
- if (definition0 == null) {
- return definition1 == null;
- } else if (definition1 == null) {
- return false;
- }
-
- return isLiteralSameCCExprString(definition0, definition1);
- }
-
- public static boolean isLiteralSameCCExprString(String definition0, String definition1) {
- definition0 = StringUtils.replaceAll(definition0, "\\s*", "");
- definition1 = StringUtils.replaceAll(definition1, "\\s*", "");
- return definition0.equalsIgnoreCase(definition1);
- }
-
private static boolean isSameCCExpr(ComputedColumnDesc existingCC, ComputedColumnDesc newCC,
AliasMapping aliasMapping) {
if (existingCC.getExpression() == null) {
@@ -547,12 +509,13 @@ public class ComputedColumnUtil {
@Override
public void handleOnSingleModelSameExpr(NDataModel existingModel, ComputedColumnDesc existingCC,
ComputedColumnDesc newCC) {
- logger.error(
- String.format(Locale.ROOT, "In model %s, computed columns %s and %s have equivalent expressions.",
- existingModel.getAlias(), existingCC.getFullName(), newCC.getFullName()));
+ String ccFullName = newCC.getFullName();
+ String errorMsg = "In model " + existingModel.getAlias() + ", computed columns " + existingCC.getFullName()
+ + " and " + ccFullName + " have equivalent expressions.";
+ logger.error(errorMsg);
String msg = MsgPicker.getMsg().getComputedColumnExpressionDuplicatedSingleModel();
throw new BadModelException(DUPLICATE_COMPUTED_COLUMN_EXPRESSION, msg,
- BadModelException.CauseType.SELF_CONFLICT_WITH_SAME_EXPRESSION, null, null, newCC.getFullName());
+ BadModelException.CauseType.SELF_CONFLICT_WITH_SAME_EXPRESSION, null, null, ccFullName);
}
}
@@ -678,19 +641,6 @@ public class ComputedColumnUtil {
}
}
- public static List<Pair<ComputedColumnDesc, NDataModel>> getExistingCCs(String modelId,
- List<NDataModel> otherModels) {
- List<Pair<ComputedColumnDesc, NDataModel>> existingCCs = Lists.newArrayList();
- for (NDataModel otherModel : otherModels) {
- if (!StringUtils.equals(otherModel.getUuid(), modelId)) {
- for (ComputedColumnDesc cc : otherModel.getComputedColumnDescs()) {
- existingCCs.add(Pair.newPair(cc, otherModel));
- }
- }
- }
- return existingCCs;
- }
-
public static List<ComputedColumnDesc> getAuthorizedCC(List<NDataModel> modelList,
Predicate<Set<String>> isColumnAuthorizedFunc) {
val authorizedCC = Lists.<ComputedColumnDesc> newArrayList();
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index ed9f738cd3..1e3ef22468 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -32,7 +32,7 @@ import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.constant.NonCustomProjectLevelConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.AutoMergeTimeEnum;
import org.apache.kylin.metadata.model.ISourceAware;
@@ -356,7 +356,7 @@ public class ProjectInstance extends RootPersistentEntity implements ISourceAwar
for (String resource : modelResource) {
String[] path = resource.split("/");
resource = path[path.length - 1];
- resource = StringUtil.dropSuffix(resource, MetadataConstants.FILE_SURFIX);
+ resource = StringHelper.dropSuffix(resource, MetadataConstants.FILE_SURFIX);
nameList.add(resource);
}
return nameList;
@@ -369,7 +369,7 @@ public class ProjectInstance extends RootPersistentEntity implements ISourceAwar
public List<String> getEmailUsers() {
String users = this.getOverrideKylinProps().get(NonCustomProjectLevelConfig.NOTIFICATION_USER_EMAILS.getValue());
if(users != null) {
- return Arrays.asList(StringUtil.split(users, ","));
+ return Arrays.asList(StringHelper.split(users, ","));
}
return new ArrayList<>();
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
index d442600a19..b2f9fc0a0e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -18,6 +18,7 @@
package org.apache.kylin.source.adhocquery;
+import java.sql.SQLException;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
@@ -37,18 +38,18 @@ public interface IPushDownRunner {
void init(KylinConfig config);
/**
- * Run an pushdown query in the source database in case Kylin cannot serve using cube.
+ * Run a push-down query in the source database in case Kylin cannot serve using cube.
*
* @param query the query statement
* @param returnRows an empty list to collect returning rows
* @param returnColumnMeta an empty list to collect metadata of returning columns
* @param project the project name
- * @throws Exception if running pushdown query fails
+ * @throws SQLException if running pushdown query fails
*/
void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta,
- String project) throws Exception;
+ String project) throws SQLException;
- default PushdownResult executeQueryToIterator(String query, String project) throws Exception {
+ default PushdownResult executeQueryToIterator(String query, String project) throws SQLException {
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
executeQuery(query, returnRows, returnColumnMeta, project);
@@ -58,13 +59,12 @@ public interface IPushDownRunner {
/**
* Run an pushdown non-query sql
*
- * @param sql the sql statement
+ * @param sql the sql statement
+ * @param project the project
*
- * @return whether the SQL is executed successfully
- *
- * @throws Exception if running pushdown fails
+ * @throws SQLException if running pushdown fails
*/
- void executeUpdate(String sql, String project) throws Exception;
+ void executeUpdate(String sql, String project) throws SQLException;
String getName();
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/planner/algorithm/AlgorithmTestBase.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/planner/algorithm/AlgorithmTestBase.java
index b1e4356db7..a4efd3523c 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/planner/algorithm/AlgorithmTestBase.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/planner/algorithm/AlgorithmTestBase.java
@@ -35,7 +35,7 @@ import java.util.Objects;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.junit.After;
import org.junit.Before;
@@ -357,7 +357,7 @@ public class AlgorithmTestBase {
StandardCharsets.UTF_8));
while ((sCurrentLine = br.readLine()) != null) {
- String[] statPair = StringUtil.split(sCurrentLine, " ");
+ String[] statPair = StringHelper.split(sCurrentLine, " ");
countMap.put(BigInteger.valueOf(Long.valueOf(statPair[0])), Long.valueOf(statPair[1]));
}
@@ -564,4 +564,4 @@ public class AlgorithmTestBase {
return scanCountMap;
}
-}
\ No newline at end of file
+}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
index 65a359e63f..65ad1e25da 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
@@ -29,8 +29,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.model.graph.DefaultJoinEdgeMatcher;
-import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.project.NProjectManager;
import org.junit.Assert;
import org.junit.Before;
@@ -351,7 +349,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
public void testColumnDescEquals() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
NTableMetadataManager manager = NTableMetadataManager.getInstance(getTestConfig(), "default");
TableDesc tableDesc = manager.getTableDesc("DEFAULT.TEST_KYLIN_FACT");
- DefaultJoinEdgeMatcher matcher = new DefaultJoinEdgeMatcher();
+ JoinsGraph.DefaultJoinEdgeMatcher matcher = new JoinsGraph.DefaultJoinEdgeMatcher();
ColumnDesc one = new ColumnDesc();
one.setTable(tableDesc);
one.setName("one");
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/ComputedColumnUtilTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/ComputedColumnUtilTest.java
index 1cdcca85fa..55f0bf990d 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/ComputedColumnUtilTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/ComputedColumnUtilTest.java
@@ -103,7 +103,7 @@ public class ComputedColumnUtilTest extends NLocalFileMetadataTestCase {
NDataModel modelNew = modelManager.getDataModelDesc(model.getUuid());
ColumnDesc column = new ColumnDesc();
column.setName("DEAL_AMOUNT");
- column.setComputedColumn("`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`");
+ column.setComputedColumnExpr("`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`");
Map<String, Set<String>> colsMapWithModel = ComputedColumnUtil.getCCUsedColsMapWithModel(modelNew, column);
Assert.assertEquals(1, colsMapWithModel.size());
Assert.assertTrue(colsMapWithModel.containsKey("DEFAULT.TEST_KYLIN_FACT"));
@@ -112,7 +112,7 @@ public class ComputedColumnUtilTest extends NLocalFileMetadataTestCase {
ColumnDesc notExistColumn = new ColumnDesc();
notExistColumn.setName("CC_NOT_EXIST");
- notExistColumn.setComputedColumn("`TEST_KYLIN_FACT`.`PRICE` * 0.95");
+ notExistColumn.setComputedColumnExpr("`TEST_KYLIN_FACT`.`PRICE` * 0.95");
try {
ComputedColumnUtil.getCCUsedColsMapWithModel(modelNew, notExistColumn);
Assert.fail();
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SampleController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SampleController.java
index 60f672d993..f13bc7f40b 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SampleController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SampleController.java
@@ -88,12 +88,14 @@ public class SampleController extends BaseController {
@ResponseBody
public EnvelopeResponse<String> getPartitionColumnFormat(@RequestParam(value = "project") String project,
@RequestParam(value = "table") String table,
- @RequestParam(value = "partition_column") String partitionColumn) throws Exception {
+ @RequestParam(value = "partition_column") String partitionColumn,
+ @RequestParam(value = "partition_expression", required = false) String partitionExpression)
+ throws Exception {
checkProjectName(project);
checkRequiredArg(TABLE, table);
checkRequiredArg("partitionColumn", partitionColumn);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
- tableService.getPartitionColumnFormat(project, table, partitionColumn), "");
+ tableService.getPartitionColumnFormat(project, table, partitionColumn, partitionExpression), "");
}
@ApiOperation(value = "samplingJobs", tags = { "AI" })
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java
index de0bbdfd6b..4932580b7d 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java
@@ -96,13 +96,14 @@ public class OpenSampleController extends BaseController {
@ResponseBody
public EnvelopeResponse<OpenPartitionColumnFormatResponse> getPartitionColumnFormat(
@RequestParam(value = "project") String project, @RequestParam(value = "table") String table,
- @RequestParam(value = "column_name") String columnName) throws Exception {
+ @RequestParam(value = "column_name") String columnName,
+ @RequestParam(value = "expression", required = false) String expression) throws Exception {
String projectName = checkProjectName(project);
checkRequiredArg(TABLE, table);
checkRequiredArg("column_name", columnName);
- String columnFormat = tableService.getPartitionColumnFormat(projectName,
- StringUtils.upperCase(table, Locale.ROOT), columnName);
+ String columnFormat = tableService.getPartitionColumnFormat(projectName, StringUtils.upperCase(table),
+ columnName, expression);
OpenPartitionColumnFormatResponse columnFormatResponse = new OpenPartitionColumnFormatResponse();
columnFormatResponse.setColumnName(columnName);
columnFormatResponse.setColumnFormat(columnFormat);
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
index 0b494a9c30..442ed3c584 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
@@ -131,7 +131,7 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(openSampleController).refreshSegments(Mockito.any(RefreshSegmentsRequest.class));
}
-
+
@Test
public void testSubmitSamplingCaseInsensitive() throws Exception {
String tableMixture = "dEFault.teST_kylIN_fact";
@@ -199,7 +199,7 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
.param("project", project).param("table", tableName).param("column_name", columnName)
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(openSampleController).getPartitionColumnFormat(project, tableName, columnName);
+ Mockito.verify(openSampleController).getPartitionColumnFormat(project, tableName, columnName, null);
}
{
@@ -216,7 +216,7 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(tableService, Mockito.times(1)).getPartitionColumnFormat(project, tableNameUppercase,
- columnName);
+ columnName, null);
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
.contentType(MediaType.APPLICATION_JSON) //
@@ -224,7 +224,7 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(tableService, Mockito.times(2)).getPartitionColumnFormat(project, tableNameUppercase,
- columnName);
+ columnName, null);
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
.contentType(MediaType.APPLICATION_JSON) //
@@ -232,7 +232,7 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(tableService, Mockito.times(3)).getPartitionColumnFormat(project, tableNameUppercase,
- columnName);
+ columnName, null);
}
}
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index dc35003be5..0e0c4a6e9f 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -76,7 +76,7 @@ import org.apache.kylin.common.scheduler.JobDiscardNotifier;
import org.apache.kylin.common.scheduler.JobReadyNotifier;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.job.common.JobUtil;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
@@ -140,6 +140,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import io.kyligence.kap.metadata.epoch.EpochManager;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.Getter;
import lombok.Setter;
@@ -1366,7 +1367,7 @@ public class JobService extends BasicService implements JobSupporter, ISmartAppl
return;
}
// The user's browser may contain multiple language preferences, such as xx,xx;ss,ss
- String language = StringUtil.dropFirstSuffix(StringUtil.dropFirstSuffix(languageToHandle, ";"), ",");
+ String language = StringHelper.dropFirstSuffix(StringHelper.dropFirstSuffix(languageToHandle, ";"), ",");
if (CHINESE_LANGUAGE.equals(language) || CHINESE_SIMPLE_LANGUAGE.equals(language)
|| CHINESE_HK_LANGUAGE.equals(language) || CHINESE_TW_LANGUAGE.equals(language)) {
ErrorCode.setMsg("cn");
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
index 176c6281a2..1a40feb5ec 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
@@ -24,12 +24,38 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
import lombok.val;
+import static org.apache.kylin.common.exception.ServerErrorCode.COLUMN_NOT_EXIST;
+import static org.apache.kylin.common.exception.ServerErrorCode.DATABASE_NOT_EXIST;
+import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER;
+import static org.apache.kylin.common.exception.ServerErrorCode.PERMISSION_DENIED;
+import static org.apache.kylin.common.exception.ServerErrorCode.SNAPSHOT_MANAGEMENT_NOT_ENABLED;
+import static org.apache.kylin.common.exception.ServerErrorCode.SNAPSHOT_NOT_EXIST;
+import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_NOT_EXIST;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_FAIL;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY;
+import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_BUILD;
+import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_REFRESH;
+import static org.apache.kylin.rest.constant.SnapshotStatus.BROKEN;
+import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.dao.JobStatisticsManager;
@@ -83,20 +109,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.kylin.common.exception.ServerErrorCode.COLUMN_NOT_EXIST;
-import static org.apache.kylin.common.exception.ServerErrorCode.DATABASE_NOT_EXIST;
-import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER;
-import static org.apache.kylin.common.exception.ServerErrorCode.PERMISSION_DENIED;
-import static org.apache.kylin.common.exception.ServerErrorCode.SNAPSHOT_MANAGEMENT_NOT_ENABLED;
-import static org.apache.kylin.common.exception.ServerErrorCode.SNAPSHOT_NOT_EXIST;
-import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_NOT_EXIST;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_FAIL;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY;
-import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_BUILD;
-import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_REFRESH;
-import static org.apache.kylin.rest.constant.SnapshotStatus.BROKEN;
-import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
-
@Component("snapshotService")
public class SnapshotService extends BasicService implements SnapshotSupporter {
@@ -158,7 +170,7 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
public JobInfoResponse buildSnapshotsInner(SnapshotRequest snapshotsRequest, boolean isRefresh,
- Set<String> needBuildSnapshotTables, Set<TableDesc> tables) {
+ Set<String> needBuildSnapshotTables, Set<TableDesc> tables) {
val project = snapshotsRequest.getProject();
val options = snapshotsRequest.getOptions();
List<String> invalidSnapshotsToBuild = new ArrayList<>();
@@ -212,15 +224,15 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
private void updateTableDesc(String project, Set<TableDesc> tables,
- Map<String, SnapshotRequest.TableOption> finalOptions) {
+ Map<String, SnapshotRequest.TableOption> finalOptions) {
NTableMetadataManager tableManager = getManager(NTableMetadataManager.class, project);
for (TableDesc tableDesc : tables) {
SnapshotRequest.TableOption option = finalOptions.get(tableDesc.getIdentity());
if (tableDesc.isSnapshotHasBroken()
- || !StringUtil.equals(option.getPartitionCol(), tableDesc.getSelectedSnapshotPartitionCol())) {
+ || !StringUtils.equals(option.getPartitionCol(), tableDesc.getSelectedSnapshotPartitionCol())) {
TableDesc newTable = tableManager.copyForWrite(tableDesc);
newTable.setSnapshotHasBroken(false);
- if (!StringUtil.equals(option.getPartitionCol(), tableDesc.getSelectedSnapshotPartitionCol())) {
+ if (!StringUtils.equals(option.getPartitionCol(), tableDesc.getSelectedSnapshotPartitionCol())) {
newTable.setSelectedSnapshotPartitionCol(option.getPartitionCol());
}
tableManager.updateTableDesc(newTable);
@@ -229,7 +241,7 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
private static void invalidSnapshotsToBuild(Map<String, SnapshotRequest.TableOption> options,
- List<String> invalidSnapshotsToBuild) {
+ List<String> invalidSnapshotsToBuild) {
for (Map.Entry<String, SnapshotRequest.TableOption> entry : options.entrySet()) {
Set<String> partitionToBuild = entry.getValue().getPartitionsToBuild();
if (partitionToBuild != null && partitionToBuild.isEmpty()) {
@@ -243,7 +255,7 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
public JobInfoResponse buildSnapshots(SnapshotRequest snapshotsRequest, boolean isRefresh,
- Set<String> needBuildSnapshotTables) {
+ Set<String> needBuildSnapshotTables) {
val project = snapshotsRequest.getProject();
checkSnapshotManualManagement(project);
Set<TableDesc> tables = checkAndGetTable(project, needBuildSnapshotTables);
@@ -440,8 +452,8 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
@Override
public Pair<List<SnapshotInfoResponse>, Integer> getProjectSnapshots(String project, String table,
- Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed,
- Pair<Integer, Integer> offsetAndLimit) {
+ Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed,
+ Pair<Integer, Integer> offsetAndLimit) {
checkSnapshotManualManagement(project);
aclEvaluate.checkProjectReadPermission(project);
NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, project);
@@ -472,8 +484,8 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
TableExtDesc tableExtDesc = nTableMetadataManager.getOrCreateTableExt(tableDesc);
Pair<Integer, Integer> countPair = getModelCount(tableDesc);
- response.add(new SnapshotInfoResponse(tableDesc, tableExtDesc, tableDesc.getSnapshotTotalRows(), countPair.getFirst(),
- countPair.getSecond(), getSnapshotJobStatus(tableDesc, executables),
+ response.add(new SnapshotInfoResponse(tableDesc, tableExtDesc, tableDesc.getSnapshotTotalRows(),
+ countPair.getFirst(), countPair.getSecond(), getSnapshotJobStatus(tableDesc, executables),
getForbiddenColumns(tableDesc)));
satisfiedTableSize.getAndIncrement();
});
@@ -487,7 +499,8 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
// Here the positive order needs to be cut from the offset position backwards
Comparator<SnapshotInfoResponse> comparator = BasicService.propertyComparator(sortBy, !isReversed);
response.sort(comparator);
- return Pair.newPair(PagingUtil.cutPage(response, offsetAndLimit.getFirst(), offsetAndLimit.getSecond()), actualTableSize);
+ return Pair.newPair(PagingUtil.cutPage(response, offsetAndLimit.getFirst(), offsetAndLimit.getSecond()),
+ actualTableSize);
}
}
@@ -500,8 +513,8 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
public List<TableDesc> getFilteredTables(NTableMetadataManager nTableMetadataManager,
- Pair<String, String> databaseAndTable, boolean canUseACLGreenChannel, Set<String> finalAuthorizedTables,
- List<AbstractExecutable> executables, Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter) {
+ Pair<String, String> databaseAndTable, boolean canUseACLGreenChannel, Set<String> finalAuthorizedTables,
+ List<AbstractExecutable> executables, Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter) {
String finalDatabase = databaseAndTable.getFirst();
String finalTable = databaseAndTable.getSecond();
return nTableMetadataManager.listAllTables().stream().filter(tableDesc -> {
@@ -523,15 +536,14 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
return true;
}
return finalAuthorizedTables.contains(tableDesc.getIdentity());
- }).filter(tableDesc -> hasLoadedSnapshot(tableDesc, executables)
- ).filter(tableDesc -> statusFilter.isEmpty() || statusFilter.contains(getSnapshotJobStatus(tableDesc, executables))
- ).filter(tableDesc -> {
- if (partitionFilter.size() != 1) {
- return true;
- }
- boolean isPartition = partitionFilter.iterator().next();
- return isPartition != (tableDesc.getSelectedSnapshotPartitionCol() == null);
- }).collect(Collectors.toList());
+ }).filter(tableDesc -> hasLoadedSnapshot(tableDesc, executables)).filter(tableDesc -> statusFilter.isEmpty()
+ || statusFilter.contains(getSnapshotJobStatus(tableDesc, executables))).filter(tableDesc -> {
+ if (partitionFilter.size() != 1) {
+ return true;
+ }
+ boolean isPartition = partitionFilter.iterator().next();
+ return isPartition != (tableDesc.getSelectedSnapshotPartitionCol() == null);
+ }).collect(Collectors.toList());
}
private Pair<Integer, Integer> getModelCount(TableDesc tableDesc) {
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
index ed87e6c628..cb2896da8f 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
@@ -95,7 +95,6 @@ import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.query.util.PushDownUtil;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.request.ModelRequest;
import org.apache.kylin.rest.request.PartitionsRefreshRequest;
@@ -199,7 +198,7 @@ public class ModelServiceBuildTest extends SourceTestCase {
ReflectionTestUtils.setField(modelBuildService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = PushDownUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.createAclInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
@@ -787,7 +786,7 @@ public class ModelServiceBuildTest extends SourceTestCase {
NDataflowUpdate dataflowUpdate = new NDataflowUpdate(dataflow.getUuid());
dataflowUpdate.setToRemoveSegs(dataflow.getSegments().toArray(new NDataSegment[dataflow.getSegments().size()]));
dataflowManager.updateDataflow(dataflowUpdate);
- val minAndMaxTime = PushDownUtil.getMaxAndMinTime(modelUpdate.getPartitionDesc().getPartitionDateColumn(),
+ val minAndMaxTime = PushDownUtil.probeMinMaxTs(modelUpdate.getPartitionDesc().getPartitionDateColumn(),
modelUpdate.getRootFactTableName(), "default");
val dateFormat = DateFormat.proposeDateFormat(minAndMaxTime.getFirst());
modelBuildService.buildSegmentsManually("default", "89af4ee2-2cdb-4b07-b39e-4c29856309aa",
@@ -807,7 +806,7 @@ public class ModelServiceBuildTest extends SourceTestCase {
Assert.assertEquals(t1, dataflow.getSegments().get(0).getSegRange().getStart());
Assert.assertEquals(t2, dataflow.getSegments().get(0).getSegRange().getEnd());
- val result = PushDownUtil.getMaxAndMinTimeWithTimeOut(modelUpdate.getPartitionDesc().getPartitionDateColumn(),
+ val result = PushDownUtil.probeMinMaxTsWithTimeout(modelUpdate.getPartitionDesc().getPartitionDateColumn(),
modelUpdate.getRootFactTableName(), "default");
Assert.assertNotNull(result);
}
diff --git a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java
index 5660e92ec9..1839d98c6b 100644
--- a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java
+++ b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java
@@ -53,7 +53,7 @@ public class PushDownRunnerSDKImpl implements IPushDownRunner {
@Override
public void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta,
- String project) throws Exception {
+ String project) throws SQLException {
query = dataSource.convertSql(query);
//extract column metadata
@@ -90,7 +90,7 @@ public class PushDownRunnerSDKImpl implements IPushDownRunner {
}
@Override
- public void executeUpdate(String sql, String project) throws Exception {
+ public void executeUpdate(String sql, String project) throws SQLException {
dataSource.executeUpdate(sql);
}
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkSourceService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkSourceService.java
index b26fcf61ac..4acf6676ae 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkSourceService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkSourceService.java
@@ -34,15 +34,15 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.source.ISourceMetadataExplorer;
-import org.apache.kylin.source.SourceFactory;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.rest.request.DDLRequest;
import org.apache.kylin.rest.response.DDLResponse;
import org.apache.kylin.rest.response.ExportTablesResponse;
import org.apache.kylin.rest.response.TableNameResponse;
+import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.apache.kylin.source.SourceFactory;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.DDLDesc;
import org.apache.spark.sql.Dataset;
@@ -272,15 +272,15 @@ public class SparkSourceService extends BasicService {
ss.sql(CREATE_VIEW_P_LINEORDER);
createdTables.add(VIEW_P_LINEORDER);
}
- log.info("Load samples {} successfully", StringUtil.join(createdTables, ","));
+ log.info("Load samples {} successfully", StringHelper.join(createdTables, ","));
} finally {
lock.unlock();
}
return createdTables;
}
- private void loadSamples(SparkSession ss, SaveMode mode, String table, String tableName,
- String db, File file, String fileName) throws IOException {
+ private void loadSamples(SparkSession ss, SaveMode mode, String table, String tableName, String db, File file,
+ String fileName) throws IOException {
String filePath = file.getAbsolutePath();
FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
String hdfsPath = String.format(Locale.ROOT, "/tmp/%s", fileName);
@@ -289,13 +289,12 @@ public class SparkSourceService extends BasicService {
File[] parquetFiles = file.listFiles();
if (parquetFiles != null) {
for (File parquetFile : parquetFiles) {
- fileSystem.copyFromLocalFile(new Path(parquetFile.getAbsolutePath()),
- new Path(hdfsPath));
+ fileSystem.copyFromLocalFile(new Path(parquetFile.getAbsolutePath()), new Path(hdfsPath));
}
}
// KC-6666, check and delete location
- String tbLocation = String.format(Locale.ROOT, "%s/%s",
- ss.catalog().getDatabase(db).locationUri(), tableName);
+ String tbLocation = String.format(Locale.ROOT, "%s/%s", ss.catalog().getDatabase(db).locationUri(),
+ tableName);
FileSystem fs = FileSystem.get(ss.sparkContext().hadoopConfiguration());
Path path = new Path(tbLocation);
if (fs.exists(path)) {
@@ -305,8 +304,7 @@ public class SparkSourceService extends BasicService {
ss.read().parquet(hdfsPath).write().mode(mode).saveAsTable(table);
} catch (Exception e) {
log.error("Load sample {} failed.", fileName, e);
- throw new IllegalStateException(String.format(Locale.ROOT, "Load sample %s failed", fileName),
- e);
+ throw new IllegalStateException(String.format(Locale.ROOT, "Load sample %s failed", fileName), e);
} finally {
fileSystem.delete(new Path(hdfsPath), false);
}
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index efa49e11b1..fbde97f9ec 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -228,20 +228,24 @@ public class TableService extends BasicService {
@Autowired
private ClusterManager clusterManager;
- public Pair<List<TableDesc>, Integer> getTableDesc(String project, boolean withExt, final String table, final String database,
- boolean isFuzzy, List<Integer> sourceType, int returnTableSize) throws IOException {
- TableDescRequest internalTableDescRequest = new TableDescRequest(project, withExt, table, database, isFuzzy, sourceType);
+ public Pair<List<TableDesc>, Integer> getTableDesc(String project, boolean withExt, final String table,
+ final String database, boolean isFuzzy, List<Integer> sourceType, int returnTableSize) throws IOException {
+ TableDescRequest internalTableDescRequest = new TableDescRequest(project, withExt, table, database, isFuzzy,
+ sourceType);
return getTableDesc(internalTableDescRequest, returnTableSize);
}
- public Pair<List<TableDesc>, Integer> getTableDesc(TableDescRequest tableDescRequest, int returnTableSize) throws IOException {
+ public Pair<List<TableDesc>, Integer> getTableDesc(TableDescRequest tableDescRequest, int returnTableSize)
+ throws IOException {
aclEvaluate.checkProjectReadPermission(tableDescRequest.getProject());
boolean streamingEnabled = getConfig().streamingEnabled();
- NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, tableDescRequest.getProject());
+ NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class,
+ tableDescRequest.getProject());
List<TableDesc> tables = Lists.newArrayList();
//get table not fuzzy,can use getTableDesc(tableName)
if (StringUtils.isNotEmpty(tableDescRequest.getTable()) && !tableDescRequest.isFuzzy()) {
- val tableDesc = nTableMetadataManager.getTableDesc(tableDescRequest.getDatabase() + "." + tableDescRequest.getTable());
+ val tableDesc = nTableMetadataManager
+ .getTableDesc(tableDescRequest.getDatabase() + "." + tableDescRequest.getTable());
if (tableDesc != null && tableDesc.isAccessible(streamingEnabled))
tables.add(tableDesc);
} else {
@@ -254,7 +258,8 @@ public class TableService extends BasicService {
if (StringUtils.isEmpty(tableDescRequest.getTable())) {
return true;
}
- return tableDesc.getName().toLowerCase(Locale.ROOT).contains(tableDescRequest.getTable().toLowerCase(Locale.ROOT));
+ return tableDesc.getName().toLowerCase(Locale.ROOT)
+ .contains(tableDescRequest.getTable().toLowerCase(Locale.ROOT));
}).filter(tableDesc -> {
// Advance the logic of filtering the table by sourceType to here
if (!tableDescRequest.getSourceType().isEmpty()) {
@@ -451,7 +456,8 @@ public class TableService extends BasicService {
return tableDescResponse;
}
- private Pair<List<TableDesc>, Integer> getTablesResponse(List<TableDesc> tables, String project, boolean withExt, int returnTableSize) {
+ private Pair<List<TableDesc>, Integer> getTablesResponse(List<TableDesc> tables, String project, boolean withExt,
+ int returnTableSize) {
List<TableDesc> descs = new ArrayList<>();
val projectManager = getManager(NProjectManager.class);
val groups = getCurrentUserGroups();
@@ -515,8 +521,7 @@ public class TableService extends BasicService {
AclTCRManager manager = getManager(AclTCRManager.class, project);
Map<Integer, AclTCR.ColumnRealRows> columnRows = Arrays.stream(rtableDesc.getExtColumns()).map(cdr -> {
int id = Integer.parseInt(cdr.getId());
- val columnRealRows = manager.getAuthorizedRows(dbTblName, cdr.getName(),
- aclTCRS);
+ val columnRealRows = manager.getAuthorizedRows(dbTblName, cdr.getName(), aclTCRS);
return new AbstractMap.SimpleEntry<>(id, columnRealRows);
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
for (String[] row : rtableDesc.getSamplingRows()) {
@@ -711,7 +716,8 @@ public class TableService extends BasicService {
() -> jobManager.addSegmentJob(new JobParam(newSegment, model, getUsername())));
}
- public String getPartitionColumnFormat(String project, String table, String partitionColumn) throws Exception {
+ public String getPartitionColumnFormat(String project, String table, String partitionColumn,
+ String partitionExpression) throws Exception {
aclEvaluate.checkProjectOperationPermission(project);
NTableMetadataManager tableManager = getManager(NTableMetadataManager.class, project);
@@ -719,7 +725,7 @@ public class TableService extends BasicService {
Preconditions.checkNotNull(tableDesc, String.format(Locale.ROOT, MsgPicker.getMsg().getTableNotFound(), table));
Set<String> columnSet = Stream.of(tableDesc.getColumns()).map(ColumnDesc::getName)
.map(str -> str.toUpperCase(Locale.ROOT)).collect(Collectors.toSet());
- if (!columnSet.contains(partitionColumn.toUpperCase(Locale.ROOT))) {
+ if (!columnSet.contains(StringUtils.upperCase(partitionColumn))) {
throw new KylinException(COLUMN_NOT_EXIST, String.format(Locale.ROOT,
"Can not find the column:%s in table:%s, project:%s", partitionColumn, table, project));
}
@@ -735,11 +741,14 @@ public class TableService extends BasicService {
mapping.forEach((key, value) -> mappingAllCaps.put(key.toUpperCase(Locale.ROOT), value));
String cell = (String) mappingAllCaps.get(partitionColumn);
return DateFormat.proposeDateFormat(cell);
+ } else if (partitionExpression == null) {
+ List<String> list = PushDownUtil.backtickQuote(partitionColumn.split("\\."));
+ String cell = PushDownUtil.probeColFormat(table, String.join(".", list), project);
+ return DateFormat.proposeDateFormat(cell);
} else {
- String cell = PushDownUtil.getFormatIfNotExist(table, partitionColumn, project);
+ String cell = PushDownUtil.probeExpFormat(table, partitionExpression, project);
return DateFormat.proposeDateFormat(cell);
}
-
} catch (KylinException e) {
throw e;
} catch (Exception e) {
@@ -1786,12 +1795,14 @@ public class TableService extends BasicService {
}
public NInitTablesResponse getProjectTables(String project, String table, int offset, int limit,
- boolean withExcluded, boolean useHiveDatabase, List<Integer> sourceType) throws Exception {
- TableDescRequest internalTableDescRequest = new TableDescRequest(project, table, offset, limit, withExcluded, sourceType);
+ boolean withExcluded, boolean useHiveDatabase, List<Integer> sourceType) throws Exception {
+ TableDescRequest internalTableDescRequest = new TableDescRequest(project, table, offset, limit, withExcluded,
+ sourceType);
return getProjectTables(internalTableDescRequest, useHiveDatabase);
}
- public NInitTablesResponse getProjectTables(TableDescRequest tableDescRequest, boolean useHiveDatabase) throws Exception {
+ public NInitTablesResponse getProjectTables(TableDescRequest tableDescRequest, boolean useHiveDatabase)
+ throws Exception {
String project = tableDescRequest.getProject();
aclEvaluate.checkProjectReadPermission(project);
NInitTablesResponse response = new NInitTablesResponse();
@@ -1825,12 +1836,14 @@ public class TableService extends BasicService {
objWithActualSize.setSecond(hiveTableNameResponses.size());
} else {
int returnTableSize = calculateTableSize(tableDescRequest.getOffset(), tableDescRequest.getLimit());
- Pair<List<TableDesc>, Integer> tableDescWithActualSize = getTableDesc(tableDescRequest, returnTableSize);
+ Pair<List<TableDesc>, Integer> tableDescWithActualSize = getTableDesc(tableDescRequest,
+ returnTableSize);
objWithActualSize.setFirst(tableDescWithActualSize.getFirst());
objWithActualSize.setSecond(tableDescWithActualSize.getSecond());
}
table = notAllowedModifyTableName;
- List<?> tablePage = PagingUtil.cutPage(objWithActualSize.getFirst(), tableDescRequest.getOffset(), tableDescRequest.getLimit());
+ List<?> tablePage = PagingUtil.cutPage(objWithActualSize.getFirst(), tableDescRequest.getOffset(),
+ tableDescRequest.getLimit());
if (!tablePage.isEmpty()) {
response.putDatabase(database, objWithActualSize.getSecond(), tablePage);
}
@@ -2013,7 +2026,7 @@ public class TableService extends BasicService {
public void refreshTable(String table, List<String> refreshed, List<String> failed) {
try {
- PushDownUtil.trySimplePushDownExecute("REFRESH TABLE " + table, null);
+ PushDownUtil.trySimplyExecute("REFRESH TABLE " + table, null);
refreshed.add(table);
} catch (Exception e) {
failed.add(table);
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/AntiFlatCheckerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/AntiFlatCheckerTest.java
index 02d0f0f185..bb7a59013c 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/AntiFlatCheckerTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/AntiFlatCheckerTest.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.junit.annotation.MetadataInfo;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.PushDownUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -42,7 +42,7 @@ class AntiFlatCheckerTest {
NDataModelManager modelManager = NDataModelManager.getInstance(kylinConfig, getProject());
NDataModel model = modelManager.getDataModelDesc(modelId);
model.getComputedColumnDescs().forEach(cc -> {
- String innerExp = QueryUtil.massageComputedColumn(model, getProject(), cc, null);
+ String innerExp = PushDownUtil.massageComputedColumn(model, getProject(), cc, null);
cc.setInnerExpression(innerExp);
});
model.init(kylinConfig, getProject(), Lists.newArrayList());
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/ColExcludedCheckerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/ColExcludedCheckerTest.java
index 82c7bac737..996bd70094 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/ColExcludedCheckerTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/metadata/model/ColExcludedCheckerTest.java
@@ -23,7 +23,7 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.junit.annotation.MetadataInfo;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.util.MetadataTestUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -44,7 +44,7 @@ class ColExcludedCheckerTest {
NDataModelManager modelManager = NDataModelManager.getInstance(kylinConfig, getProject());
NDataModel model = modelManager.getDataModelDesc(modelId);
model.getComputedColumnDescs().forEach(cc -> {
- String innerExp = QueryUtil.massageComputedColumn(model, getProject(), cc, null);
+ String innerExp = PushDownUtil.massageComputedColumn(model, getProject(), cc, null);
cc.setInnerExpression(innerExp);
});
model.init(kylinConfig, getProject(), Lists.newArrayList());
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java
index 22072e7d93..268045a66a 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java
@@ -36,7 +36,7 @@ import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
-import org.apache.kylin.query.util.PushDownUtil;
+import org.apache.kylin.query.KylinTestBase;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
@@ -253,13 +253,13 @@ public class NBadQueryAndPushDownTest extends NLocalWithSparkSessionTest {
int offset, SQLException sqlException, boolean isForced) throws Exception {
populateSSWithCSVData(KylinConfig.getInstanceFromEnv(), prjName, SparderEnv.getSparkSession());
String pushdownSql = ExecAndComp.removeDataBaseInSql(sql);
- String massagedSql = QueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), pushdownSql, limit, offset);
+ String massagedSql = QueryUtil.appendLimitOffset(prjName, pushdownSql, limit, offset);
QueryParams queryParams = new QueryParams(prjName, massagedSql, "DEFAULT", BackdoorToggles.getPrepareOnly(),
sqlException, isForced);
queryParams.setSelect(true);
queryParams.setLimit(limit);
queryParams.setOffset(offset);
- Pair<List<List<String>>, List<SelectedColumnMeta>> result = PushDownUtil.tryPushDownQuery(queryParams);
+ Pair<List<List<String>>, List<SelectedColumnMeta>> result = KylinTestBase.tryPushDownQuery(queryParams);
if (result == null) {
throw sqlException;
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index c538c0dd1e..be3d732e46 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -51,8 +51,11 @@ import org.apache.kylin.util.ExecAndComp;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableList;
import com.google.common.io.Files;
+import lombok.val;
+
/**
*/
public class KylinTestBase extends NLocalFileMetadataTestCase {
@@ -134,13 +137,22 @@ public class KylinTestBase extends NLocalFileMetadataTestCase {
public Pair<List<List<String>>, List<SelectedColumnMeta>> tryPushDownSelectQuery(String project, String sql,
String defaultSchema, SQLException sqlException, boolean isPrepare, boolean isForced) throws Exception {
- String massagedSql = QueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), sql, 0, 0);
+ String massagedSql = QueryUtil.appendLimitOffset(project, sql, 0, 0);
QueryParams queryParams = new QueryParams(project, massagedSql, defaultSchema, isPrepare, sqlException,
isForced);
queryParams.setSelect(true);
queryParams.setLimit(0);
queryParams.setOffset(0);
- return PushDownUtil.tryPushDownQuery(queryParams);
+ return tryPushDownQuery(queryParams);
+ }
+
+ public static Pair<List<List<String>>, List<SelectedColumnMeta>> tryPushDownQuery(QueryParams queryParams)
+ throws Exception {
+ val results = PushDownUtil.tryIterQuery(queryParams);
+ if (results == null) {
+ return null;
+ }
+ return new Pair<>(ImmutableList.copyOf(results.getRows()), results.getColumnMetas());
}
// end of execute
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java b/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
index c3fd6c4df3..801ae07b65 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
@@ -45,6 +45,7 @@ import org.apache.kylin.metadata.query.StructField;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.data.QueryResult;
import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.spark.sql.Dataset;
@@ -175,7 +176,7 @@ public class ExecAndComp {
QueryParams queryParams = new QueryParams(prj, compareSql, "default", false);
queryParams.setKylinConfig(NProjectManager.getProjectConfig(prj));
- String afterConvert = QueryUtil.massagePushDownSql(queryParams);
+ String afterConvert = PushDownUtil.massagePushDownSql(queryParams);
// Table schema comes from csv and DATABASE.TABLE is not supported.
String sqlForSpark = removeDataBaseInSql(afterConvert);
val ds = querySparkSql(sqlForSpark);
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 1dfd998028..b685328a42 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -42,7 +42,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.rest.request.AWSTableLoadRequest;
@@ -117,7 +117,8 @@ public class NTableController extends NBasicController {
"AI" }, notes = "Update Param: is_fuzzy, page_offset, page_size; Update Response: no format!")
@GetMapping(value = "", produces = { HTTP_VND_APACHE_KYLIN_JSON })
@ResponseBody
- public EnvelopeResponse<Map<String, Object>> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt,
+ public EnvelopeResponse<Map<String, Object>> getTableDesc(
+ @RequestParam(value = "ext", required = false) boolean withExt,
@RequestParam(value = "project") String project,
@RequestParam(value = "table", required = false) String table,
@RequestParam(value = "database", required = false) String database,
@@ -133,7 +134,8 @@ public class NTableController extends NBasicController {
int returnTableSize = calculateTableSize(offset, limit);
TableDescRequest tableDescRequest = new TableDescRequest(project, table, database, withExt, isFuzzy,
Pair.newPair(offset, limit), Collections.singletonList(sourceType));
- Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(tableDescRequest, returnTableSize);
+ Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(tableDescRequest,
+ returnTableSize);
// Finally, the results are processed based on the paging parameters and returned to the front-end UI,
// where the results table to be processed each time is getting longer as the number of paging increases
Map<String, Object> mockDataResponse = setCustomDataResponse("tables", tableDescWithActualSize, offset, limit);
@@ -155,8 +157,8 @@ public class NTableController extends NBasicController {
@RequestParam(value = "source_type", required = false, defaultValue = "9") List<Integer> sourceType)
throws Exception {
checkProjectName(project);
- TableDescRequest tableDescRequest = new TableDescRequest(project, table, "", withExt, isFuzzy,
- offset, limit, sourceType, withExcluded);
+ TableDescRequest tableDescRequest = new TableDescRequest(project, table, "", withExt, isFuzzy, offset, limit,
+ sourceType, withExcluded);
NInitTablesResponse projectTables = tableService.getProjectTables(tableDescRequest, false);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, projectTables, "");
}
@@ -230,7 +232,7 @@ public class NTableController extends NBasicController {
LoadTableResponse loadTableResponse = new LoadTableResponse();
if (ArrayUtils.isNotEmpty(tableLoadRequest.getTables())) {
- StringUtil.toUpperCaseArray(tableLoadRequest.getTables(), tableLoadRequest.getTables());
+ StringHelper.toUpperCaseArray(tableLoadRequest.getTables(), tableLoadRequest.getTables());
LoadTableResponse loadByTable = tableExtService.loadDbTables(tableLoadRequest.getTables(),
tableLoadRequest.getProject(), false);
loadTableResponse.getFailed().addAll(loadByTable.getFailed());
@@ -238,7 +240,7 @@ public class NTableController extends NBasicController {
}
if (ArrayUtils.isNotEmpty(tableLoadRequest.getDatabases())) {
- StringUtil.toUpperCaseArray(tableLoadRequest.getDatabases(), tableLoadRequest.getDatabases());
+ StringHelper.toUpperCaseArray(tableLoadRequest.getDatabases(), tableLoadRequest.getDatabases());
LoadTableResponse loadByDb = tableExtService.loadDbTables(tableLoadRequest.getDatabases(),
tableLoadRequest.getProject(), true);
loadTableResponse.getFailed().addAll(loadByDb.getFailed());
@@ -351,8 +353,8 @@ public class NTableController extends NBasicController {
@RequestParam(value = "page_offset", required = false, defaultValue = "0") Integer offset,
@RequestParam(value = "page_size", required = false, defaultValue = "10") Integer limit) throws Exception {
String projectName = checkProjectName(project);
- NInitTablesResponse data = tableService.getProjectTables(projectName, table, offset, limit, true,
- true, Collections.emptyList());
+ NInitTablesResponse data = tableService.getProjectTables(projectName, table, offset, limit, true, true,
+ Collections.emptyList());
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, data, "");
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
index 1f0e6aa271..a1f41efe75 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
@@ -34,7 +34,7 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.AWSTableLoadRequest;
@@ -152,8 +152,8 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetTableDesc() throws Exception {
- TableDescRequest mockTableDescRequest = new TableDescRequest("default", "", "DEFAULT", false,
- true, Pair.newPair(0, 10), Collections.singletonList(9));
+ TableDescRequest mockTableDescRequest = new TableDescRequest("default", "", "DEFAULT", false, true,
+ Pair.newPair(0, 10), Collections.singletonList(9));
Mockito.when(tableService.getTableDesc(mockTableDescRequest, 10)).thenReturn(Pair.newPair(mockTables(), 10));
@@ -294,8 +294,8 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase {
}
private void initMockito(LoadTableResponse loadTableResponse, TableLoadRequest tableLoadRequest) throws Exception {
- StringUtil.toUpperCaseArray(tableLoadRequest.getTables(), tableLoadRequest.getTables());
- StringUtil.toUpperCaseArray(tableLoadRequest.getDatabases(), tableLoadRequest.getDatabases());
+ StringHelper.toUpperCaseArray(tableLoadRequest.getTables(), tableLoadRequest.getTables());
+ StringHelper.toUpperCaseArray(tableLoadRequest.getDatabases(), tableLoadRequest.getDatabases());
Mockito.when(tableExtService.loadDbTables(tableLoadRequest.getTables(), "default", false))
.thenReturn(loadTableResponse);
Mockito.when(tableExtService.loadDbTables(tableLoadRequest.getDatabases(), "default", true))
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index b7a21eb955..147b413710 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -38,6 +38,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.dialect.CalciteSqlDialect;
import org.apache.calcite.sql.dialect.HiveSqlDialect;
import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.collections.CollectionUtils;
@@ -96,6 +97,7 @@ import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.request.ModelRequest;
import org.apache.kylin.rest.response.BuildIndexResponse;
import org.apache.kylin.rest.response.SimplifiedMeasure;
@@ -131,7 +133,7 @@ public class ModelSemanticHelper extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(ModelSemanticHelper.class);
private final ExpandableMeasureUtil expandableMeasureUtil = new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = PushDownUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.createAclInfo(model.getProject(), getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
ComputedColumnEvalUtil.evaluateExprAndType(model, ccDesc);
@@ -158,13 +160,22 @@ public class ModelSemanticHelper extends BasicService {
logger.error("Parse json failed...", e);
throw new KylinException(CommonErrorCode.FAILED_PARSE_JSON, e);
}
+
+ Map<String, TableDesc> allTablesMap = getManager(NTableMetadataManager.class, modelRequest.getProject())
+ .getAllTablesMap();
+ List<ComputedColumnDesc> ccList = dataModel.getComputedColumnDescs();
+ if (!ccList.isEmpty()) {
+ String factTableIdentity = dataModel.getRootFactTableName();
+ TableDesc tableDesc = allTablesMap.get(factTableIdentity);
+ TableDesc extendTable = tableDesc.appendColumns(ComputedColumnUtil.createComputedColumns(ccList, tableDesc),
+ true);
+ allTablesMap.put(factTableIdentity, extendTable);
+ }
dataModel.setUuid(modelRequest.getUuid() != null ? modelRequest.getUuid() : RandomUtil.randomUUIDStr());
dataModel.setProject(modelRequest.getProject());
dataModel.setAllMeasures(convertMeasure(simplifiedMeasures));
dataModel.setAllNamedColumns(convertNamedColumns(modelRequest.getProject(), dataModel, modelRequest));
-
- dataModel.initJoinDesc(KylinConfig.getInstanceFromEnv(),
- getManager(NTableMetadataManager.class, modelRequest.getProject()).getAllTablesMap());
+ dataModel.initJoinDesc(KylinConfig.getInstanceFromEnv(), allTablesMap);
convertNonEquiJoinCond(dataModel, modelRequest);
dataModel.setModelType(dataModel.getModelTypeFromTable());
return dataModel;
@@ -431,7 +442,7 @@ public class ModelSemanticHelper extends BasicService {
SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAliasName, newAliasName);
SqlNode sqlNode = CalciteParser.getExpNode(filterCondition);
sqlNode.accept(modifyAlias);
- String newFilterCondition = sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString();
+ String newFilterCondition = sqlNode.toSqlString(CalciteSqlDialect.DEFAULT, true).toString();
model.setFilterCondition(newFilterCondition);
}
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index ba966e6d28..a7dd882b84 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -73,6 +73,7 @@ import static org.apache.kylin.metadata.model.FunctionDesc.PARAMETER_TYPE_COLUMN
import java.io.IOException;
import java.math.BigDecimal;
+import java.sql.SQLException;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -94,7 +95,6 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.calcite.sql.SqlDialect;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.dialect.CalciteSqlDialect;
@@ -105,7 +105,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
@@ -128,7 +127,7 @@ import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil;
import org.apache.kylin.job.SecondStorageJobParamUtil;
import org.apache.kylin.job.common.SegmentUtil;
@@ -200,7 +199,6 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.streaming.KafkaConfig;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
@@ -1738,7 +1736,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
QueryParams queryParams = new QueryParams(project, flatTableSql, "default", false);
queryParams.setKylinConfig(prjInstance.getConfig());
queryParams.setAclInfo(AclPermissionUtil.createAclInfo(project, getCurrentUserGroups()));
- String pushdownSql = QueryUtil.massagePushDownSql(queryParams);
+ String pushdownSql = PushDownUtil.massagePushDownSql(queryParams);
ss.sql(pushdownSql);
}
} catch (Exception e) {
@@ -2264,8 +2262,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
}
- public String probeDateFormatIfNotExist(String project, NDataModel modelDesc) throws Exception {
- val partitionDesc = modelDesc.getPartitionDesc();
+ public String probeDateFormatIfNotExist(String project, NDataModel model) throws SQLException {
+ PartitionDesc partitionDesc = model.getPartitionDesc();
if (PartitionDesc.isEmptyPartitionDesc(partitionDesc)
|| StringUtils.isNotEmpty(partitionDesc.getPartitionDateFormat()))
return "";
@@ -2275,9 +2273,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return partitionDesc.getPartitionDateColumn();
}
- String partitionColumn = modelDesc.getPartitionDesc().getPartitionDateColumnRef().getExpressionInSourceDB();
-
- val date = PushDownUtil.getFormatIfNotExist(modelDesc.getRootFactTableName(), partitionColumn, project);
+ String partitionColumn = model.getPartitionDesc().getPartitionDateColumnRef().getBackTickExp();
+ String date = PushDownUtil.probeColFormat(model.getRootFactTableName(), partitionColumn, project);
return DateFormat.proposeDateFormat(date);
}
@@ -2297,7 +2294,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
String dateFormat = desc.getPartitionDateFormat();
Preconditions.checkArgument(StringUtils.isNotEmpty(dateFormat) && StringUtils.isNotEmpty(partitionColumn));
- val minAndMaxTime = PushDownUtil.getMaxAndMinTimeWithTimeOut(partitionColumn, table, project);
+ val minAndMaxTime = PushDownUtil.probeMinMaxTsWithTimeout(partitionColumn, table, project);
return new Pair<>(DateFormat.getFormattedDate(minAndMaxTime.getFirst(), dateFormat),
DateFormat.getFormattedDate(minAndMaxTime.getSecond(), dateFormat));
@@ -2628,7 +2625,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
model.init(getConfig(), project, getManager(NDataModelManager.class, project).getCCRelatedModels(model));
model.getComputedColumnDescs().forEach(cc -> {
- String innerExp = QueryUtil.massageComputedColumn(model, project, cc, null);
+ String innerExp = PushDownUtil.massageComputedColumn(model, project, cc, null);
cc.setInnerExpression(innerExp);
});
@@ -2655,7 +2652,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
MsgPicker.getMsg().getccOnAntiFlattenLookup(), antiFlattenLookup));
}
ComputedColumnDesc.simpleParserCheck(cc.getExpression(), model.getAliasMap().keySet());
- String innerExpression = QueryUtil.massageComputedColumn(model, project, cc, aclInfo);
+ String innerExpression = PushDownUtil.massageComputedColumn(model, project, cc, aclInfo);
cc.setInnerExpression(innerExpression);
//check by data source, this could be slow
@@ -2757,10 +2754,10 @@ public class ModelService extends AbstractModelService implements TableModelSupp
// Update CC expression from query transformers
QueryContext.AclInfo aclInfo = AclPermissionUtil.createAclInfo(project, getCurrentUserGroups());
for (ComputedColumnDesc ccDesc : model.getComputedColumnDescs()) {
- String ccExpression = QueryUtil.massageComputedColumn(model, project, ccDesc, aclInfo);
+ String ccExpression = PushDownUtil.massageComputedColumn(model, project, ccDesc, aclInfo);
ccDesc.setInnerExpression(ccExpression);
TblColRef tblColRef = model.findColumn(ccDesc.getTableAlias(), ccDesc.getColumnName());
- tblColRef.getColumnDesc().setComputedColumn(ccExpression);
+ tblColRef.getColumnDesc().setComputedColumnExpr(ccExpression);
}
ComputedColumnEvalUtil.evalDataTypeOfCCInBatch(model, model.getComputedColumnDescs());
@@ -3045,7 +3042,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return baseIndexResponse;
}, project);
} catch (TransactionException te) {
- Throwable root = ExceptionUtils.getCause(te);
+ Throwable root = te.getCause();
if (root instanceof RuntimeException) {
throw (RuntimeException) root;
}
@@ -3439,24 +3436,24 @@ public class ModelService extends AbstractModelService implements TableModelSupp
String pattitions = props.get("kylin.engine.spark-conf.spark.sql.shuffle.partitions");
String memory = props.get("kylin.engine.spark-conf.spark.executor.memory");
String baseCuboidAllowed = props.get("kylin.cube.aggrgroup.is-base-cuboid-always-valid");
- if (null != cores && !StringUtil.validateNumber(cores)) {
+ if (null != cores && !StringHelper.validateNumber(cores)) {
throw new KylinException(INVALID_PARAMETER,
String.format(Locale.ROOT, MsgPicker.getMsg().getInvalidIntegerFormat(), "spark.executor.cores"));
}
- if (null != instances && !StringUtil.validateNumber(instances)) {
+ if (null != instances && !StringHelper.validateNumber(instances)) {
throw new KylinException(INVALID_PARAMETER, String.format(Locale.ROOT,
MsgPicker.getMsg().getInvalidIntegerFormat(), "spark.executor.instances"));
}
- if (null != pattitions && !StringUtil.validateNumber(pattitions)) {
+ if (null != pattitions && !StringHelper.validateNumber(pattitions)) {
throw new KylinException(INVALID_PARAMETER, String.format(Locale.ROOT,
MsgPicker.getMsg().getInvalidIntegerFormat(), "spark.sql.shuffle.partitions"));
}
if (null != memory
- && (!memory.endsWith("g") || !StringUtil.validateNumber(memory.substring(0, memory.length() - 1)))) {
+ && (!memory.endsWith("g") || !StringHelper.validateNumber(memory.substring(0, memory.length() - 1)))) {
throw new KylinException(INVALID_PARAMETER,
String.format(Locale.ROOT, MsgPicker.getMsg().getInvalidMemorySize(), "spark.executor.memory"));
}
- if (null != baseCuboidAllowed && !StringUtil.validateBoolean(baseCuboidAllowed)) {
+ if (null != baseCuboidAllowed && !StringHelper.validateBoolean(baseCuboidAllowed)) {
throw new KylinException(INVALID_PARAMETER, String.format(Locale.ROOT,
MsgPicker.getMsg().getInvalidBooleanFormat(), "is-base-cuboid-always-valid"));
}
@@ -3600,17 +3597,15 @@ public class ModelService extends AbstractModelService implements TableModelSupp
*
* @param model
*/
- @VisibleForTesting
- public void massageModelFilterCondition(final NDataModel model) {
+ void massageModelFilterCondition(final NDataModel model) {
if (StringUtils.isEmpty(model.getFilterCondition())) {
return;
}
- String massagedFilterCond = QueryUtil.massageExpression(model, model.getProject(), model.getFilterCondition(),
- AclPermissionUtil.createAclInfo(model.getProject(), getCurrentUserGroups()), false);
-
- String filterConditionWithTableName = addTableNameIfNotExist(massagedFilterCond, model);
-
+ String filterConditionWithTableName = addTableNameIfNotExist(model.getFilterCondition(), model);
+ QueryContext.AclInfo aclInfo = AclPermissionUtil.createAclInfo(model.getProject(), getCurrentUserGroups());
+ // validate as soon as possible
+ PushDownUtil.massageExpression(model, model.getProject(), model.getFilterCondition(), aclInfo);
model.setFilterCondition(filterConditionWithTableName);
}
@@ -3648,9 +3643,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
MsgPicker.getMsg().getFilterConditionOnAntiFlattenLookup(), antiFlatLookup));
}
}
- String exp = sqlNode
- .toSqlString(new CalciteSqlDialect(SqlDialect.EMPTY_CONTEXT.withIdentifierQuoteString("`")), true)
- .toString();
+ String exp = sqlNode.toSqlString(CalciteSqlDialect.DEFAULT, true).toString();
return CalciteParser.normalize(exp);
}
@@ -4171,7 +4164,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
for (ComputedColumnDesc cc : model.getComputedColumnDescs()) {
String innerExp = cc.getInnerExpression();
if (cc.getExpression().equalsIgnoreCase(innerExp)) {
- innerExp = QueryUtil.massageComputedColumn(model, project, cc, null);
+ innerExp = PushDownUtil.massageComputedColumn(model, project, cc, null);
}
cc.setInnerExpression(innerExp);
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 9c8293e8b9..cb82fc5921 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -159,6 +159,7 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
@@ -285,7 +286,7 @@ public class ModelServiceTest extends SourceTestCase {
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = PushDownUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.createAclInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
@@ -3267,8 +3268,8 @@ public class ModelServiceTest extends SourceTestCase {
modelRequest.getPartitionDesc().setPartitionDateFormat("yyyy-MM-dd");
String filterCond = "trans_id = 0 and TEST_KYLIN_FACT.order_id < 100 and DEAL_AMOUNT > 123";
- String expectedFilterCond = "(((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) AND (`TEST_KYLIN_FACT`.`ORDER_ID` <"
- + " 100)) AND ((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))";
+ String expectedFilterCond = "(((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) "
+ + "AND (\"TEST_KYLIN_FACT\".\"ORDER_ID\" < 100)) AND (\"TEST_KYLIN_FACT\".\"DEAL_AMOUNT\" > 123))";
modelRequest.setFilterCondition(filterCond);
val newModel = modelService.createModel(modelRequest.getProject(), modelRequest);
@@ -3292,8 +3293,7 @@ public class ModelServiceTest extends SourceTestCase {
modelRequest.setUuid(null);
String filterCond = "\"day\" = 0 and \"123TABLE\".\"day#\" = 1 and \"中文列\" = 1";
- String expectedFilterCond = "(((`123TABLE`.`DAY` = 0) AND (`123TABLE`.`day#` = 1)) AND (`123TABLE`"
- + ".`中文列` = 1))";
+ String expectedFilterCond = "(((\"123TABLE\".\"DAY\" = 0) AND (\"123TABLE\".\"day#\" = 1)) AND (\"123TABLE\".\"中文列\" = 1))";
modelRequest.setFilterCondition(filterCond);
val newModel = modelService.createModel(modelRequest.getProject(), modelRequest);
@@ -3542,10 +3542,8 @@ public class ModelServiceTest extends SourceTestCase {
String originSql = "trans_id = 0 and TEST_KYLIN_FACT.order_id < 100 and DEAL_AMOUNT > 123";
model.setFilterCondition(originSql);
modelService.massageModelFilterCondition(model);
- Assert.assertEquals(
- "(((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) AND " + "(`TEST_KYLIN_FACT`.`ORDER_ID` < 100)) AND "
- + "((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))",
- model.getFilterCondition());
+ Assert.assertEquals("(((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) AND (\"TEST_KYLIN_FACT\".\"ORDER_ID\" < 100)) "
+ + "AND (\"TEST_KYLIN_FACT\".\"DEAL_AMOUNT\" > 123))", model.getFilterCondition());
}
@Test
@@ -3577,7 +3575,7 @@ public class ModelServiceTest extends SourceTestCase {
final NDataModel model1 = modelManager.getDataModelDesc(modelId);
model1.setFilterCondition("TIMESTAMPDIFF(DAY, CURRENT_DATE, TEST_KYLIN_FACT.\"CURRENT_DATE\") >= 0");
modelService.massageModelFilterCondition(model1);
- Assert.assertEquals("(TIMESTAMPDIFF(DAY, CURRENT_DATE(), `TEST_KYLIN_FACT`.`CURRENT_DATE`) >= 0)",
+ Assert.assertEquals("(TIMESTAMPDIFF(DAY, CURRENT_DATE(), \"TEST_KYLIN_FACT\".\"CURRENT_DATE\") >= 0)",
model1.getFilterCondition());
}
@@ -3593,10 +3591,8 @@ public class ModelServiceTest extends SourceTestCase {
String originSql = "trans_id = 0 and TEST_ORDER.order_id < 100 and DEAL_AMOUNT > 123";
model.setFilterCondition(originSql);
modelService.massageModelFilterCondition(model);
- Assert.assertEquals(
- "(((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) " + "AND (`TEST_ORDER`.`ORDER_ID` < 100)) "
- + "AND ((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))",
- model.getFilterCondition());
+ Assert.assertEquals("(((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) AND (\"TEST_ORDER\".\"ORDER_ID\" < 100)) "
+ + "AND (\"TEST_KYLIN_FACT\".\"DEAL_AMOUNT\" > 123))", model.getFilterCondition());
}
@Test
@@ -3624,10 +3620,11 @@ public class ModelServiceTest extends SourceTestCase {
NDataModel model = modelManager.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
String originSql = "trans_id = 0 and TEST_KYLIN_FACT.order_id < 100";
String newSql = modelService.addTableNameIfNotExist(originSql, model);
- Assert.assertEquals("((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) AND (`TEST_KYLIN_FACT`.`ORDER_ID` < 100))", newSql);
+ Assert.assertEquals("((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) AND (\"TEST_KYLIN_FACT\".\"ORDER_ID\" < 100))",
+ newSql);
originSql = "trans_id between 1 and 10";
newSql = modelService.addTableNameIfNotExist(originSql, model);
- Assert.assertEquals("(`TEST_KYLIN_FACT`.`TRANS_ID` BETWEEN 1 AND 10)", newSql);
+ Assert.assertEquals("(\"TEST_KYLIN_FACT\".\"TRANS_ID\" BETWEEN 1 AND 10)", newSql);
modelManager.updateDataModel(model.getUuid(), copyForWrite -> {
List<JoinTableDesc> joinTables = copyForWrite.getJoinTables();
@@ -3685,11 +3682,11 @@ public class ModelServiceTest extends SourceTestCase {
}
@Test
- public void testComputedColumnNameCheck_PreProcessBeforeModelSave_ExceptionWhenCCNameIsSameWithColumnInLookupTable() {
+ public void testCheckingCcNameIsSameWithLookupColNameBeforeModelSaveThenThrowException() {
expectedEx.expect(KylinException.class);
- expectedEx.expectMessage("Can’t validate the expression \"TEST_KYLIN_FACT.NEST2\" (computed column: "
- + "TEST_KYLIN_FACT.NEST1 * 12). Please check the expression, or try again later.");
+ expectedEx.expectMessage("Can’t validate the expression \"TEST_KYLIN_FACT.SITE_ID\" (computed column: "
+ + "nvl(TEST_SITES.SITE_ID)). Please check the expression, or try again later.");
String tableIdentity = "DEFAULT.TEST_KYLIN_FACT";
String columnName = "SITE_ID";
String expression = "nvl(TEST_SITES.SITE_ID)";
@@ -3703,13 +3700,17 @@ public class ModelServiceTest extends SourceTestCase {
String project = "default";
NDataModelManager dataModelManager = modelService.getManager(NDataModelManager.class, "default");
NDataModel model = dataModelManager.getDataModelDesc("741ca86a-1f13-46da-a59f-95fb68615e3a");
- model.getComputedColumnDescs().add(ccDesc);
+ model.getComputedColumnDescs().add(0, ccDesc);
modelService.preProcessBeforeModelSave(model, project);
}
@Test
- public void testCheckCCNameAmbiguity() {
+ public void testCheckingCcNameIsSameWithLookupColNameWhenCheckingCCThenThrowException() {
+
+ expectedEx.expect(KylinException.class);
+ expectedEx.expectMessage("Can’t validate the expression \"TEST_KYLIN_FACT.SITE_ID\" (computed column: "
+ + "nvl(TEST_SITES.SITE_ID)). Please check the expression, or try again later.");
String tableIdentity = "DEFAULT.TEST_KYLIN_FACT";
String columnName = "SITE_ID";
String expression = "nvl(TEST_SITES.SITE_ID)";
@@ -3720,19 +3721,16 @@ public class ModelServiceTest extends SourceTestCase {
ccDesc.setExpression(expression);
ccDesc.setDatatype(dataType);
+ String project = "default";
NDataModelManager dataModelManager = modelService.getManager(NDataModelManager.class, "default");
NDataModel model = dataModelManager.getDataModelDesc("741ca86a-1f13-46da-a59f-95fb68615e3a");
- model.getComputedColumnDescs().add(ccDesc);
+ model.getComputedColumnDescs().add(0, ccDesc);
- modelService.checkCCNameAmbiguity(model);
+ modelService.checkComputedColumn(model, project, null);
}
@Test
- public void testComputedColumnNameCheck_CheckCC_ExceptionWhenCCNameIsSameWithColumnInLookupTable() {
-
- expectedEx.expect(KylinException.class);
- expectedEx.expectMessage("Can’t validate the expression \"TEST_KYLIN_FACT.NEST2\" (computed column: "
- + "TEST_KYLIN_FACT.NEST1 * 12). Please check the expression, or try again later.");
+ public void testCheckCCNameAmbiguity() {
String tableIdentity = "DEFAULT.TEST_KYLIN_FACT";
String columnName = "SITE_ID";
String expression = "nvl(TEST_SITES.SITE_ID)";
@@ -3743,12 +3741,11 @@ public class ModelServiceTest extends SourceTestCase {
ccDesc.setExpression(expression);
ccDesc.setDatatype(dataType);
- String project = "default";
NDataModelManager dataModelManager = modelService.getManager(NDataModelManager.class, "default");
NDataModel model = dataModelManager.getDataModelDesc("741ca86a-1f13-46da-a59f-95fb68615e3a");
model.getComputedColumnDescs().add(ccDesc);
- modelService.checkComputedColumn(model, project, null);
+ modelService.checkCCNameAmbiguity(model);
}
private NDataSegment mockSegment() {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index 5864a7e5de..b9d9b71857 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -208,23 +208,22 @@ public class TableServiceTest extends CSVSourceTestCase {
List<Integer> sourceType = new ArrayList<>();
sourceType.add(1); // Kafka table
sourceType.add(9); // Hive table
- List<TableDesc> tableDesc = tableService.getTableDesc("default", true, "", "DEFAULT", true,
- sourceType, 12).getFirst();
+ List<TableDesc> tableDesc = tableService.getTableDesc("default", true, "", "DEFAULT", true, sourceType, 12)
+ .getFirst();
Assert.assertEquals(12, tableDesc.size());
- List<TableDesc> tableDesc2 = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", false,
- sourceType, 10).getFirst();
+ List<TableDesc> tableDesc2 = tableService
+ .getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", false, sourceType, 10).getFirst();
Assert.assertEquals(1, tableDesc2.size());
- List<TableDesc> tables3 = tableService.getTableDesc("default", true, "", "", true,
- sourceType, 100).getFirst();
+ List<TableDesc> tables3 = tableService.getTableDesc("default", true, "", "", true, sourceType, 100).getFirst();
Assert.assertEquals(21, tables3.size());
- List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", true,
- sourceType, 10).getFirst();
+ List<TableDesc> tables = tableService
+ .getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", true, sourceType, 10).getFirst();
Assert.assertEquals("TEST_KYLIN_FACT", tables.get(0).getName());
Assert.assertEquals(5633024, ((TableDescResponse) tables.get(0)).getStorageSize());
Assert.assertEquals(0, ((TableDescResponse) tables.get(0)).getTotalRecords());
- List<TableDesc> table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true,
- sourceType, 10).getFirst();
+ List<TableDesc> table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true, sourceType, 10)
+ .getFirst();
Assert.assertEquals("TEST_COUNTRY", table2.get(0).getName());
Assert.assertEquals(0L, ((TableDescResponse) table2.get(0)).getStorageSize());
@@ -233,25 +232,22 @@ public class TableServiceTest extends CSVSourceTestCase {
countryTable.setLastSnapshotPath("cannot/find/it");
manager.updateTableDesc(countryTable);
- table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true,
- sourceType, 10).getFirst();
+ table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true, sourceType, 10).getFirst();
Assert.assertEquals("TEST_COUNTRY", table2.get(0).getName());
Assert.assertEquals(0L, ((TableDescResponse) table2.get(0)).getStorageSize());
// get a not existing table desc
- tableDesc = tableService.getTableDesc("default", true, "not_exist_table", "DEFAULT", false,
- sourceType, 10).getFirst();
+ tableDesc = tableService.getTableDesc("default", true, "not_exist_table", "DEFAULT", false, sourceType, 10)
+ .getFirst();
Assert.assertEquals(0, tableDesc.size());
- tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true,
- sourceType, 10).getFirst();
+ tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true, sourceType, 10).getFirst();
Assert.assertEquals(2, tableDesc.size());
val tableMetadataManager = getInstance(getTestConfig(), "streaming_test");
var tableDesc1 = tableMetadataManager.getTableDesc("DEFAULT.SSB_TOPIC");
Assert.assertTrue(tableDesc1.isAccessible(getTestConfig().streamingEnabled()));
getTestConfig().setProperty("kylin.streaming.enabled", "false");
- tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true,
- sourceType, 10).getFirst();
+ tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true, sourceType, 10).getFirst();
Assert.assertEquals(0, tableDesc.size());
// check kafka table
Assert.assertFalse(tableDesc1.isAccessible(getTestConfig().streamingEnabled()));
@@ -289,8 +285,8 @@ public class TableServiceTest extends CSVSourceTestCase {
Assert.assertEquals(1, newTableExt.getAllColumnStats().size());
// call api to check tableDescResponse has the correct value
- final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ final List<TableDesc> tables = tableService
+ .getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
TableDescResponse t = (TableDescResponse) tables.get(0);
@@ -333,8 +329,8 @@ public class TableServiceTest extends CSVSourceTestCase {
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString()))
.thenReturn(false);
- List<TableDesc> tableExtList = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT",
- true, Collections.emptyList(), 10).getFirst();
+ List<TableDesc> tableExtList = tableService
+ .getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
Assert.assertEquals(0, ((TableDescResponse) tableExtList.get(0)).getSamplingRows().size());
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
@@ -406,8 +402,8 @@ public class TableServiceTest extends CSVSourceTestCase {
Assert.assertEquals("float", confirmedTableDesc.getColumns()[2].getDatatype());
// call api to check tableDescResponse has the correct value
- final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ final List<TableDesc> tables = tableService
+ .getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
TableDescResponse t = (TableDescResponse) tables.get(0);
@@ -495,8 +491,8 @@ public class TableServiceTest extends CSVSourceTestCase {
@Test
public void testLoadTableToProject() throws IOException {
- List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ List<TableDesc> tables = tableService
+ .getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
TableDesc nTableDesc = new TableDesc(tables.get(0));
TableExtDesc tableExt = new TableExtDesc();
tableExt.setIdentity("DEFAULT.TEST_COUNTRY");
@@ -509,8 +505,8 @@ public class TableServiceTest extends CSVSourceTestCase {
public void testLoadTableToProjectWithS3Role() throws IOException {
getTestConfig().setProperty("kylin.env.use-dynamic-S3-role-credential-in-table", "true");
assert !SparderEnv.getSparkSession().conf().contains(String.format(S3AUtil.ROLE_ARN_KEY_FORMAT, "testbucket"));
- List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ List<TableDesc> tables = tableService
+ .getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
TableDesc nTableDesc = new TableDesc(tables.get(0));
TableExtDesc tableExt = new TableExtDesc();
tableExt.setIdentity("DEFAULT.TEST_COUNTRY");
@@ -738,7 +734,7 @@ public class TableServiceTest extends CSVSourceTestCase {
tableDesc.setTableType(TableDesc.TABLE_TYPE_VIEW);
tableMgr.updateTableDesc(tableDesc);
try {
- tableService.getPartitionColumnFormat("default", table, "CAL_DT");
+ tableService.getPartitionColumnFormat("default", table, "CAL_DT", null);
Assert.fail();
} catch (Exception e) {
Assert.assertEquals(MsgPicker.getMsg().getViewDateFormatDetectionError(), e.getMessage());
@@ -752,7 +748,7 @@ public class TableServiceTest extends CSVSourceTestCase {
testGetBatchLoadTablesBefore();
final String table = "DEFAULT.TEST_KYLIN_FACT";
try {
- tableService.getPartitionColumnFormat("default", table, "CAL_DT");
+ tableService.getPartitionColumnFormat("default", table, "CAL_DT", null);
Assert.fail();
} catch (Exception e) {
Assert.assertEquals(MsgPicker.getMsg().getPushdownPartitionFormatError(), e.getMessage());
@@ -774,8 +770,8 @@ public class TableServiceTest extends CSVSourceTestCase {
private void testSetPartitionKeyWithoutException() throws Exception {
tableService.setPartitionKey("DEFAULT.TEST_KYLIN_FACT", "default", "CAL_DT", "yyyy-MM-dd");
- List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ List<TableDesc> tables = tableService
+ .getTableDesc("default", false, "", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
//test set fact and table list order by fact
Assert.assertTrue(tables.get(0).getName().equals("TEST_KYLIN_FACT") && tables.get(0).isIncrementLoading());
}
@@ -869,7 +865,7 @@ public class TableServiceTest extends CSVSourceTestCase {
private void testgetPartitionColumnFormat() throws Exception {
// Test on batch table
- String format = tableService.getPartitionColumnFormat("default", "DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
+ String format = tableService.getPartitionColumnFormat("default", "DEFAULT.TEST_KYLIN_FACT", "CAL_DT", null);
Assert.assertEquals("yyyy-MM-dd", format);
// Test on streaming table
@@ -897,7 +893,8 @@ public class TableServiceTest extends CSVSourceTestCase {
when(kafkaServiceMock.decodeMessage(any())).thenReturn(mockResp);
when(kafkaServiceMock.parserMessage(any(String.class), any(), any(String.class))).thenReturn(parseMap);
- String format2 = tableService.getPartitionColumnFormat("default", "DEFAULT.STREAMING_TABLE", "MINUTE_START");
+ String format2 = tableService.getPartitionColumnFormat("default", "DEFAULT.STREAMING_TABLE", "MINUTE_START",
+ null);
Assert.assertEquals("yyyy-MM-dd HH:mm:ss", format2);
when(kafkaServiceMock.getMessages(any(), any(String.class))).thenCallRealMethod();
@@ -922,8 +919,8 @@ public class TableServiceTest extends CSVSourceTestCase {
public void testSetTop() throws IOException {
TopTableRequest topTableRequest = mockTopTableRequest();
tableService.setTop(topTableRequest.getTable(), topTableRequest.getProject(), topTableRequest.isTop());
- List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ List<TableDesc> tables = tableService
+ .getTableDesc("default", false, "", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
Assert.assertTrue(tables.get(0).isTop());
}
@@ -1139,35 +1136,29 @@ public class TableServiceTest extends CSVSourceTestCase {
NInitTablesResponse response;
overwriteSystemProp("kylin.source.load-hive-tablename-enabled", "false");
- response = tableService.getProjectTables("default", "SSB.SS", 0, 14, true, true,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", "SSB.SS", 0, 14, true, true, Collections.emptyList());
Assert.assertEquals(0, response.getDatabases().size());
- response = tableService.getProjectTables("default", "SSB.CU", 0, 14, true, true,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", "SSB.CU", 0, 14, true, true, Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(2, response.getDatabases().get(0).getTables().size());
- response = tableService.getProjectTables("default", "", 0, 14, true, true,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", "", 0, 14, true, true, Collections.emptyList());
Assert.assertEquals(3, response.getDatabases().size());
Assert.assertEquals(21,
response.getDatabases().get(0).getTables().size() + response.getDatabases().get(1).getTables().size()
+ response.getDatabases().get(2).getTables().size());
- response = tableService.getProjectTables("default", "TEST", 0, 14, true, true,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", "TEST", 0, 14, true, true, Collections.emptyList());
Assert.assertEquals(2, response.getDatabases().size());
Assert.assertEquals(13,
response.getDatabases().get(0).getTables().size() + response.getDatabases().get(1).getTables().size());
- response = tableService.getProjectTables("default", "EDW.", 0, 14, true, true,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", "EDW.", 0, 14, true, true, Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(3, response.getDatabases().get(0).getTables().size());
- response = tableService.getProjectTables("default", "EDW.", 0, 14, true, false,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", "EDW.", 0, 14, true, false, Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(3, response.getDatabases().get(0).getTables().size());
@@ -1176,12 +1167,10 @@ public class TableServiceTest extends CSVSourceTestCase {
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(1, response.getDatabases().get(0).getTables().size());
- response = tableService.getProjectTables("default", ".TEST_ORDER", 0, 14, true, false,
- Collections.emptyList());
+ response = tableService.getProjectTables("default", ".TEST_ORDER", 0, 14, true, false, Collections.emptyList());
Assert.assertEquals(0, response.getDatabases().size());
- response = tableService.getProjectTables("default", "", 0, 14, true, true,
- Collections.singletonList(9));
+ response = tableService.getProjectTables("default", "", 0, 14, true, true, Collections.singletonList(9));
Assert.assertEquals(3, response.getDatabases().size());
}
@@ -1323,18 +1312,18 @@ public class TableServiceTest extends CSVSourceTestCase {
String warehousePath = getTestConfig().exportToProperties()
.getProperty("kylin.storage.columnar.spark-conf.spark.sql.warehouse.dir").substring(5)
+ "/test_kylin_refresh/";
- PushDownUtil.trySimplePushDownExecute("drop table if exists test_kylin_refresh", null);
- PushDownUtil.trySimplePushDownExecute("create table test_kylin_refresh (word string) STORED AS PARQUET", null);
- PushDownUtil.trySimplePushDownExecute("insert into test_kylin_refresh values ('a')", null);
- PushDownUtil.trySimplePushDownExecute("insert into test_kylin_refresh values ('c')", null);
- PushDownUtil.trySimplePushDownExecute("select * from test_kylin_refresh", null);
+ PushDownUtil.trySimplyExecute("drop table if exists test_kylin_refresh", null);
+ PushDownUtil.trySimplyExecute("create table test_kylin_refresh (word string) STORED AS PARQUET", null);
+ PushDownUtil.trySimplyExecute("insert into test_kylin_refresh values ('a')", null);
+ PushDownUtil.trySimplyExecute("insert into test_kylin_refresh values ('c')", null);
+ PushDownUtil.trySimplyExecute("select * from test_kylin_refresh", null);
CliCommandExecutor.CliCmdExecResult res = command.execute("ls " + warehousePath, null, null);
val files = Arrays.stream(res.getCmd().split("\n")).filter(file -> file.endsWith("parquet"))
.collect(Collectors.toList());
command.execute("rm " + warehousePath + files.get(0), null, null);
try {
- PushDownUtil.trySimplePushDownExecute("select * from test_kylin_refresh", null);
+ PushDownUtil.trySimplyExecute("select * from test_kylin_refresh", null);
Assert.fail();
} catch (Exception e) {
Assert.assertTrue(e.getMessage().contains("REFRESH TABLE tableName"));
@@ -1343,7 +1332,7 @@ public class TableServiceTest extends CSVSourceTestCase {
HashMap<String, Object> request = Maps.newHashMap();
request.put("tables", Collections.singletonList("test_kylin_refresh"));
TableRefresh refreshRes = tableService.refreshSingleCatalogCache(request);
- PushDownUtil.trySimplePushDownExecute("select * from test_kylin_refresh", null);
+ PushDownUtil.trySimplyExecute("select * from test_kylin_refresh", null);
Assert.assertEquals(1, refreshRes.getRefreshed().size());
Assert.assertEquals("test_kylin_refresh", refreshRes.getRefreshed().get(0));
SparderEnv.getSparkSession().stop();
@@ -1407,8 +1396,8 @@ public class TableServiceTest extends CSVSourceTestCase {
tableExt.setJodID("949afe5d-0221-420f-92db-cdd91cb31ac8");
tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
- List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ List<TableDesc> tables = tableService
+ .getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertEquals("949afe5d-0221-420f-92db-cdd91cb31ac8", ((TableDescResponse) tables.get(0)).getJodID());
@@ -1437,18 +1426,20 @@ public class TableServiceTest extends CSVSourceTestCase {
public void testGetTableDescByType() {
String project = "streaming_test";
try {
- val tableDescs = tableService.getTableDesc(project, true, "", "default", true,
- Collections.singletonList(1), 10).getFirst();
+ val tableDescs = tableService
+ .getTableDesc(project, true, "", "default", true, Collections.singletonList(1), 10).getFirst();
Assert.assertNotNull(tableDescs);
- val tableDescs1 = tableService.getTableDesc(project, true, "P_LINEORDER_STREAMING", "ssb", true,
- Collections.singletonList(1), 10).getFirst();
+ val tableDescs1 = tableService
+ .getTableDesc(project, true, "P_LINEORDER_STREAMING", "ssb", true, Collections.singletonList(1), 10)
+ .getFirst();
Assert.assertEquals(1, tableDescs1.size());
val tableDesc1 = tableDescs1.get(0);
Assert.assertEquals(tableDesc1.getTableAlias(), tableDesc1.getKafkaConfig().getBatchTable());
- val tableDescs2 = tableService.getTableDesc(project, true, "LINEORDER_HIVE", "SSB", false,
- Collections.singletonList(9), 10).getFirst();
+ val tableDescs2 = tableService
+ .getTableDesc(project, true, "LINEORDER_HIVE", "SSB", false, Collections.singletonList(9), 10)
+ .getFirst();
Assert.assertEquals(1, tableDescs2.size());
val tableDesc2 = tableDescs2.get(0);
Assert.assertEquals(tableDesc2.getTableAlias(), tableDesc2.getIdentity());
@@ -1463,8 +1454,7 @@ public class TableServiceTest extends CSVSourceTestCase {
String project = "streaming_test";
try {
List<Integer> sourceTypes = Arrays.asList(1, 9);
- val tableDescs2 = tableService.getTableDesc(project, true, "", "SSB", false,
- sourceTypes, 10).getFirst();
+ val tableDescs2 = tableService.getTableDesc(project, true, "", "SSB", false, sourceTypes, 10).getFirst();
assert tableDescs2.stream().anyMatch(tableDesc -> tableDesc.getSourceType() == 1);
assert tableDescs2.stream().anyMatch(tableDesc -> tableDesc.getSourceType() == 9);
} catch (Exception e) {
@@ -1549,8 +1539,8 @@ public class TableServiceTest extends CSVSourceTestCase {
tableExt.setColumnStats(Lists.newArrayList(col1));
tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
- final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
- Collections.emptyList(), 10).getFirst();
+ final List<TableDesc> tables = tableService
+ .getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true, Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
TableDescResponse t = (TableDescResponse) tables.get(0);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index ddbca4afbe..9f3be8de77 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -25,11 +25,16 @@ import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringJoiner;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.ws.rs.BadRequestException;
@@ -48,15 +53,23 @@ import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.realization.RoutingIndicatorException;
import org.apache.kylin.query.exception.NoAuthorizedColsError;
import org.apache.kylin.query.security.AccessDeniedException;
+import org.apache.kylin.source.adhocquery.IPushDownConverter;
import org.apache.kylin.source.adhocquery.IPushDownRunner;
import org.apache.kylin.source.adhocquery.PushdownResult;
import org.codehaus.commons.compiler.CompileException;
@@ -64,34 +77,45 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-
-import lombok.val;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
public class PushDownUtil {
- private static final Logger logger = LoggerFactory.getLogger(PushDownUtil.class);
+ private static final Logger logger = LoggerFactory.getLogger("query");
+
+ // sql hint "/*+ MODEL_PRIORITY({cube_name}) */"
+ private static final Pattern SQL_HINT_PATTERN = Pattern
+ .compile("/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/");
+ public static final String DEFAULT_SCHEMA = "DEFAULT";
+ private static final String CC_SPLITTER = "'##CC_PUSH_DOWN_TOKEN##'";
private static final ExecutorService asyncExecutor = Executors.newCachedThreadPool();
+ private static final Map<String, IPushDownConverter> PUSH_DOWN_CONVERTER_MAP = Maps.newConcurrentMap();
+
+ static {
+ String[] classNames = KylinConfig.getInstanceFromEnv().getPushDownConverterClassNames();
+ for (String clz : classNames) {
+ try {
+ IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(clz);
+ PUSH_DOWN_CONVERTER_MAP.put(clz, converter);
+ } catch (Exception e) {
+ logger.error("Failed to init push-down converter of the sys-config: {}", clz);
+ }
+ }
+ }
private PushDownUtil() {
}
- public static PushdownResult tryPushDownQueryToIterator(QueryParams queryParams) throws Exception {
+ public static PushdownResult tryIterQuery(QueryParams queryParams) throws SQLException {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- val prjManager = NProjectManager.getInstance(kylinConfig);
- val prj = prjManager.getProject(queryParams.getProject());
String sql = queryParams.getSql();
String project = queryParams.getProject();
- kylinConfig = prj.getConfig();
- if (!kylinConfig.isPushDownEnabled()) {
- SQLException sqlException = queryParams.getSqlException();
- if (queryParams.isForcedToPushDown() || (sqlException != null
- && sqlException.getMessage().contains(QueryContext.ROUTE_USE_FORCEDTOTIEREDSTORAGE))) {
- throw new KylinException(QueryErrorCode.INVALID_PARAMETER_PUSH_DOWN,
- MsgPicker.getMsg().getDisablePushDownPrompt());
- }
+
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(queryParams.getProject());
+ if (!projectConfig.isPushDownEnabled()) {
+ checkPushDownIncapable(queryParams);
return null;
}
if (queryParams.isSelect()) {
@@ -106,26 +130,23 @@ public class PushDownUtil {
logger.info("Kylin cannot support non-select queries, routing to other engines");
}
- IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
- runner.init(kylinConfig, project);
+ IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(projectConfig.getPushDownRunnerClassName());
+ runner.init(projectConfig, project);
logger.debug("Query Pushdown runner {}", runner);
// set pushdown engine in query context
- String pushdownEngine;
+
// for file source
- int sourceType = kylinConfig.getManager(NProjectManager.class).getProject(queryParams.getProject())
- .getSourceType();
- if (sourceType == ISourceAware.ID_SPARK && KapConfig.getInstanceFromEnv().isCloud()) {
- pushdownEngine = QueryContext.PUSHDOWN_OBJECT_STORAGE;
- } else {
- pushdownEngine = runner.getName();
- }
- QueryContext.current().setPushdownEngine(pushdownEngine);
+ int sourceType = projectConfig.getDefaultSource();
+ String engine = sourceType == ISourceAware.ID_SPARK && KapConfig.getInstanceFromEnv().isCloud()
+ ? QueryContext.PUSHDOWN_OBJECT_STORAGE
+ : runner.getName();
+ QueryContext.current().setPushdownEngine(engine);
- queryParams.setKylinConfig(kylinConfig);
+ queryParams.setKylinConfig(projectConfig);
queryParams.setSql(sql);
try {
- sql = QueryUtil.massagePushDownSql(queryParams);
+ sql = massagePushDownSql(queryParams);
} catch (NoAuthorizedColsError e) {
// on no authorized cols found, return empty result
return PushdownResult.emptyResult();
@@ -143,12 +164,171 @@ public class PushDownUtil {
return PushdownResult.emptyResult();
}
- public static Pair<String, String> getMaxAndMinTimeWithTimeOut(String partitionColumn, String table, String project)
- throws Exception {
+ private static void checkPushDownIncapable(QueryParams queryParams) {
+ SQLException sqlException = queryParams.getSqlException();
+ if (queryParams.isForcedToPushDown() || (sqlException != null
+ && sqlException.getMessage().contains(QueryContext.ROUTE_USE_FORCEDTOTIEREDSTORAGE))) {
+ throw new KylinException(QueryErrorCode.INVALID_PARAMETER_PUSH_DOWN,
+ MsgPicker.getMsg().getDisablePushDownPrompt());
+ }
+ }
+
+ public static String massageComputedColumn(NDataModel model, String project, ComputedColumnDesc cc,
+ QueryContext.AclInfo aclInfo) {
+ return massageExpression(model, project, cc.getExpression(), aclInfo);
+ }
+
+ public static String massageExpression(NDataModel model, String project, String expression,
+ QueryContext.AclInfo aclInfo) {
+ String ccSql = expandComputedColumnExp(model, project, expression);
+ QueryParams queryParams = new QueryParams(project, ccSql, PushDownUtil.DEFAULT_SCHEMA, false);
+ queryParams.setKylinConfig(NProjectManager.getProjectConfig(project));
+ queryParams.setAclInfo(aclInfo);
+ String s = massagePushDownSql(queryParams);
+ return s.substring("select ".length(), s.indexOf(CC_SPLITTER) - 2).trim();
+ }
+
+ public static String massagePushDownSql(QueryParams queryParams) {
+ String sql = queryParams.getSql();
+ sql = QueryUtil.trimRightSemiColon(sql);
+ sql = SQL_HINT_PATTERN.matcher(sql).replaceAll("");
+
+ List<IPushDownConverter> pushDownConverters = fetchConverters(queryParams.getKylinConfig());
+ if (logger.isDebugEnabled()) {
+ logger.debug("All used push-down converters are: {}", pushDownConverters.stream()
+ .map(c -> c.getClass().getCanonicalName()).collect(Collectors.joining(",")));
+ }
+ for (IPushDownConverter converter : pushDownConverters) {
+ QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of " + converter.getClass(),
+ "Current step: Massage push-down sql. ");
+ sql = converter.convert(sql, queryParams.getProject(), queryParams.getDefaultSchema());
+ }
+ sql = replaceEscapedQuote(sql);
+ return sql;
+ }
+
+ static List<IPushDownConverter> fetchConverters(KylinConfig kylinConfig) {
+ List<IPushDownConverter> converters = Lists.newArrayList();
+ for (String clz : kylinConfig.getPushDownConverterClassNames()) {
+ if (PUSH_DOWN_CONVERTER_MAP.containsKey(clz)) {
+ converters.add(PUSH_DOWN_CONVERTER_MAP.get(clz));
+ } else {
+ try {
+ IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(clz);
+ PUSH_DOWN_CONVERTER_MAP.put(clz, converter);
+ converters.add(PUSH_DOWN_CONVERTER_MAP.get(clz));
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to init pushdown converter", e);
+ }
+ }
+ }
+ return converters;
+ }
+
+ public static String expandComputedColumnExp(NDataModel model, String project, String expression) {
+ StringBuilder forCC = new StringBuilder();
+ forCC.append("select ").append(expression).append(" ,").append(CC_SPLITTER) //
+ .append(" FROM ") //
+ .append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity());
+ appendJoinStatement(model, forCC, false);
+
+ String ccSql = KeywordDefaultDirtyHack.transform(forCC.toString());
+ try {
+ // massage nested CC for drafted model
+ Map<String, NDataModel> modelMap = Maps.newHashMap();
+ modelMap.put(model.getUuid(), model);
+ ccSql = RestoreFromComputedColumn.convertWithGivenModels(ccSql, project, DEFAULT_SCHEMA, modelMap);
+ } catch (Exception e) {
+ logger.warn("Failed to massage SQL expression [{}] with input model {}", ccSql, model.getUuid(), e);
+ }
+ return ccSql;
+ }
+
+ public static void appendJoinStatement(NDataModel model, StringBuilder sql, boolean singleLine) {
+ final String sep = singleLine ? " " : "\n";
+ Set<TableRef> dimTableCache = Sets.newHashSet();
+ sql.append(sep);
+ for (JoinTableDesc lookupDesc : model.getJoinTables()) {
+ JoinDesc join = lookupDesc.getJoin();
+ TableRef dimTable = lookupDesc.getTableRef();
+ if (join == null || org.apache.commons.lang3.StringUtils.isEmpty(join.getType())
+ || dimTableCache.contains(dimTable)) {
+ continue;
+ }
+
+ TblColRef[] pk = join.getPrimaryKeyColumns();
+ TblColRef[] fk = join.getForeignKeyColumns();
+ if (pk.length != fk.length) {
+ throw new IllegalStateException("Invalid join condition of lookup table: " + lookupDesc);
+ }
+ String joinType = join.getType().toUpperCase(Locale.ROOT);
+
+ sql.append(joinType).append(" JOIN ").append(doubleQuote(dimTable)) //
+ .append(" as ").append(StringHelper.doubleQuote(dimTable.getAlias())) //
+ .append(sep).append("ON ");
+
+ if (pk.length == 0 && join.getNonEquiJoinCondition() != null) {
+ sql.append(join.getNonEquiJoinCondition().getExpr());
+ dimTableCache.add(dimTable);
+ } else {
+ sql.append(concatEqualJoinCondition(pk, fk, sep));
+ dimTableCache.add(dimTable);
+ }
+ }
+ }
+
+ private static String doubleQuote(TableRef tableRef) {
+ TableDesc table = tableRef.getTableDesc();
+ return StringHelper.doubleQuote(table.getDatabase()) + "." + StringHelper.doubleQuote(table.getName());
+ }
+
+ private static String concatEqualJoinCondition(TblColRef[] pk, TblColRef[] fk, String sep) {
+ StringJoiner joiner = new StringJoiner(" AND ", "", sep);
+ for (int i = 0; i < pk.length; i++) {
+ String s = fk[i].getDoubleQuoteExp() + " = " + pk[i].getDoubleQuoteExp();
+ joiner.add(s);
+ }
+ return joiner.toString();
+ }
+
+ /**
+ * Replace the escaped quote {@code ''} with {@code \'}. <br/>
+ * For example: <br/>
+ * the origin sql is {@code select 'a''b' from t} <br/>
+ * the replaced sql is {@code select 'a\'b' from t}
+ * @param sql the input sql
+ * @return replaced sql
+ */
+ static String replaceEscapedQuote(String sql) {
+ boolean inStrVal = false;
+ boolean needTransfer = false;
+ char[] res = sql.toCharArray();
+ for (int i = 0; i < res.length; i++) {
+ if (res[i] == '\'') {
+ if (inStrVal) {
+ if (needTransfer) {
+ res[i - 1] = '\\';
+ needTransfer = false;
+ } else {
+ needTransfer = true;
+ }
+ } else {
+ inStrVal = true;
+ }
+ } else if (needTransfer) {
+ inStrVal = false;
+ needTransfer = false;
+ }
+ }
+ return new String(res);
+ }
+
+ public static Pair<String, String> probeMinMaxTsWithTimeout(String partitionColumn, String table, String project)
+ throws ExecutionException, InterruptedException {
Future<Pair<String, String>> pushDownTask = asyncExecutor.submit(() -> {
try {
- return getMaxAndMinTime(partitionColumn, table, project);
+ return probeMinMaxTs(partitionColumn, table, project);
} catch (Exception e) {
logger.error("Failed to get partition column latest data range by push down!", e);
if (e instanceof KylinException) {
@@ -158,9 +338,9 @@ public class PushDownUtil {
return null;
});
- Pair<String, String> pushdownResult;
+ Pair<String, String> result;
try {
- pushdownResult = pushDownTask.get(30, TimeUnit.SECONDS);
+ result = pushDownTask.get(30, TimeUnit.SECONDS);
} catch (TimeoutException e) {
pushDownTask.cancel(true);
throw new KylinTimeoutException("The query exceeds the set time limit of "
@@ -168,18 +348,17 @@ public class PushDownUtil {
+ "s. Current step: Getting latest data range by push down. ");
}
- return pushdownResult;
+ return result;
}
- public static Pair<String, String> getMaxAndMinTime(String partitionColumn, String table, String project)
- throws Exception {
- Pair<String, String> pair = addBackTickForIdentity(table, partitionColumn);
- String sql = String.format(Locale.ROOT, "select min(%s), max(%s) from %s", pair.getSecond(), pair.getSecond(),
- pair.getFirst());
- Pair<String, String> result = new Pair<>();
- // pushdown
- List<List<String>> returnRows = PushDownUtil.selectPartitionColumn(sql, table, project).getFirst();
+ public static Pair<String, String> probeMinMaxTs(String partitionColumn, String table, String project)
+ throws SQLException {
+ String t = String.join(".", backtickQuote(table.split("\\.")));
+ String pc = String.join(".", backtickQuote(partitionColumn.split("\\.")));
+ String sql = String.format(Locale.ROOT, "select min(%s), max(%s) from %s", pc, pc, t);
+ Pair<String, String> result = new Pair<>();
+ List<List<String>> returnRows = probePartitionColInfo(sql, table, project).getFirst();
if (returnRows.isEmpty() || returnRows.get(0).get(0) == null || returnRows.get(0).get(1) == null)
throw new BadRequestException(String.format(Locale.ROOT, MsgPicker.getMsg().getNoDataInTable(), table));
@@ -194,75 +373,72 @@ public class PushDownUtil {
}
/**
- * Use push down engine to select partition column
+ * Use push-down engine to detect partition column info.
*/
- public static Pair<List<List<String>>, List<SelectedColumnMeta>> selectPartitionColumn(String sql, String table,
- String project) throws Exception {
+ public static Pair<List<List<String>>, List<SelectedColumnMeta>> probePartitionColInfo(String sql, String table,
+ String project) throws SQLException {
NTableMetadataManager tableMgr = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
final TableDesc tableDesc = tableMgr.getTableDesc(table);
if (tableDesc.isView()) {
throw new KylinException(VIEW_PARTITION_DATE_FORMAT_DETECTION_FORBIDDEN,
MsgPicker.getMsg().getViewDateFormatDetectionError());
}
- val prjManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
- val prj = prjManager.getProject(project);
- val kylinConfig = prj.getConfig();
+
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
- // pushdown
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
IPushDownRunner runner = (IPushDownRunner) ClassUtil
- .newInstance(kylinConfig.getPartitionCheckRunnerClassNameWithDefaultValue());
- runner.init(kylinConfig, project);
+ .newInstance(projectConfig.getDefaultPartitionCheckerClassName());
+ runner.init(projectConfig, project);
runner.executeQuery(sql, returnRows, returnColumnMeta, project);
return Pair.newPair(returnRows, returnColumnMeta);
}
- public static void trySimplePushDownExecute(String sql, String project) throws Exception {
+ public static void trySimplyExecute(String sql, String project) throws SQLException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
runner.init(kylinConfig, project);
runner.executeUpdate(sql, project);
}
- public static String getFormatIfNotExist(String table, String partitionColumn, String project) throws Exception {
- Pair<String, String> pair = addBackTickForIdentity(table, partitionColumn);
- String sql = String.format(Locale.ROOT, "select %s from %s where %s is not null limit 1", pair.getSecond(),
- pair.getFirst(), pair.getSecond());
+ public static String probeColFormat(String table, String partitionColumn, String project) throws SQLException {
+ String t = String.join(".", backtickQuote(table.split("\\.")));
+ String sql = String.format(Locale.ROOT, "select %s from %s where %s is not null limit 1", partitionColumn, t,
+ partitionColumn);
+ return probe(sql, table, project);
+ }
+
+ public static String probeExpFormat(String project, String table, String expression) throws SQLException {
+ String t = String.join(".", backtickQuote(table.split("\\.")));
+ String sql = String.format(Locale.ROOT, "select %s from %s limit 1", expression, t);
+ return probe(sql, table, project);
+ }
- // push down
- List<List<String>> returnRows = PushDownUtil.selectPartitionColumn(sql, table, project).getFirst();
- if (CollectionUtils.isEmpty(returnRows) || CollectionUtils.isEmpty(returnRows.get(0)))
+ private static String probe(String sql, String table, String project) throws SQLException {
+ List<List<String>> returnRows = probePartitionColInfo(sql, table, project).getFirst();
+ if (CollectionUtils.isEmpty(returnRows) || CollectionUtils.isEmpty(returnRows.get(0))) {
throw new KylinException(EMPTY_TABLE,
String.format(Locale.ROOT, MsgPicker.getMsg().getNoDataInTable(), table));
-
+ }
return returnRows.get(0).get(0);
}
+ public static List<String> backtickQuote(String[] arr) {
+ return Arrays.stream(arr).map(StringHelper::backtickQuote).collect(Collectors.toList());
+ }
+
private static boolean isExpectedCause(SQLException sqlException) {
Preconditions.checkArgument(sqlException != null);
Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
//SqlValidatorException is not an excepted exception in the origin design.But in the multi pass scene,
//query pushdown may create tables, and the tables are not in the model, so will throw SqlValidatorException.
- if (rootCause instanceof KylinTimeoutException) {
- return false;
- }
-
- if (rootCause instanceof AccessDeniedException) {
+ if (rootCause instanceof KylinTimeoutException || rootCause instanceof AccessDeniedException) {
return false;
- }
-
- if (rootCause instanceof RoutingIndicatorException) {
- return true;
- }
-
- if (rootCause instanceof CalciteNotSupportException) {
- return true;
- }
-
- if (rootCause instanceof CompileException) {
+ } else if (rootCause instanceof RoutingIndicatorException || rootCause instanceof CalciteNotSupportException
+ || rootCause instanceof CompileException) {
return true;
}
@@ -280,27 +456,4 @@ public class PushDownUtil {
}
return start;
}
-
- /**
- *
- * @param queryParams
- * @return
- * @throws Exception
- */
- public static Pair<List<List<String>>, List<SelectedColumnMeta>> tryPushDownQuery(QueryParams queryParams)
- throws Exception {
- val results = tryPushDownQueryToIterator(queryParams);
- if (results == null) {
- return null;
- }
- return new Pair<>(ImmutableList.copyOf(results.getRows()), results.getColumnMetas());
- }
-
- protected static Pair<String, String> addBackTickForIdentity(String table, String partitionColumn) {
- String tableName = Arrays.stream(table.split("\\.")).map(s -> "`" + s + "`").collect(Collectors.joining("."));
- String partitionColumnName = Arrays.stream(partitionColumn.split("\\.")).map(s -> "`" + s + "`")
- .collect(Collectors.joining("."));
- return Pair.newPair(tableName, partitionColumnName);
- }
-
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
index f5940b6dfb..ab4fb87d2e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
@@ -105,14 +105,14 @@ public class QueryAliasMatcher {
String tableAlias = namesOfIdentifier.get(1);
String colName = namesOfIdentifier.get(2);
ColumnRowType columnRowType = alias2CRT.get(tableAlias);
- Preconditions.checkState(columnRowType != null, "Alias {} is not defined", tableAlias);
+ Preconditions.checkState(columnRowType != null, "Alias %s is not defined", tableAlias);
return columnRowType == QueryAliasMatcher.SUBQUERY_TAG ? null : columnRowType.getColumnByName(colName);
} else if (namesOfIdentifier.size() == 2) {
// tableAlias.colName
String tableAlias = namesOfIdentifier.get(0);
String colName = namesOfIdentifier.get(1);
ColumnRowType columnRowType = alias2CRT.get(tableAlias);
- Preconditions.checkState(columnRowType != null, "Alias {} is not defined", tableAlias);
+ Preconditions.checkState(columnRowType != null, "Alias %s is not defined", tableAlias);
return columnRowType == QueryAliasMatcher.SUBQUERY_TAG ? null : columnRowType.getColumnByName(colName);
} else if (namesOfIdentifier.size() == 1) {
// only colName
@@ -500,9 +500,9 @@ public class QueryAliasMatcher {
return null;
}
- private TblColRef resolveComputedColumnRef(SqlCall call, String... tableCandidates) {
+ private TblColRef resolveComputedColumnRef(SqlCall call) {
foundCC = true;
- String table = findComputedColumnTable(call, tableCandidates);
+ String table = findComputedColumnTable(call);
ColumnDesc columnDesc = new ColumnDesc("-1", RandomUtil.randomUUIDStr(), "string", "", null, null,
call.toSqlString(CalciteSqlDialect.DEFAULT).getSql());
TableRef tableRef = alias2CRT.get(table).getColumnByIndex(0).getTableRef();
@@ -510,16 +510,16 @@ public class QueryAliasMatcher {
return TblColRef.columnForUnknownModel(tableRef, columnDesc);
}
- private String findComputedColumnTable(SqlCall call, final String... tableCandidates) {
+ private String findComputedColumnTable(SqlCall call) {
final String[] result = new String[1];
SqlBasicVisitor<SqlNode> visitor = new SqlBasicVisitor<SqlNode>() {
@Override
public SqlNode visit(SqlIdentifier sqlIdentifier) {
TblColRef colRef = resolveTblColRef(sqlIdentifier, alias2CRT);
- for (String table : tableCandidates) {
- if (alias2CRT.get(table).getAllColumns().contains(colRef)) {
- result[0] = table;
+ for (Map.Entry<String, ColumnRowType> aliasMap : alias2CRT.entrySet()) {
+ if (aliasMap.getValue().getAllColumns().contains(colRef)) {
+ result[0] = aliasMap.getKey();
return sqlIdentifier;
}
}
@@ -548,18 +548,15 @@ public class QueryAliasMatcher {
if ((operand0 instanceof SqlIdentifier || operand0 instanceof SqlCall)
&& (operand1 instanceof SqlIdentifier || operand1 instanceof SqlCall)) {
- int numOfAlias = alias2CRT.size();
String pkAlias = Iterables.getLast(alias2CRT.keySet());
- String fkAlias = Iterables.get(alias2CRT.keySet(), numOfAlias - 2);
-
// sqlCall maybe used as join condition, which need to
// translate as CC
TblColRef tblColRef0 = operand0 instanceof SqlIdentifier
? resolveTblColRef((SqlIdentifier) operand0, alias2CRT)
- : resolveComputedColumnRef((SqlCall) operand0, pkAlias, fkAlias);
+ : resolveComputedColumnRef((SqlCall) operand0);
TblColRef tblColRef1 = operand1 instanceof SqlIdentifier
? resolveTblColRef((SqlIdentifier) operand1, alias2CRT)
- : resolveComputedColumnRef((SqlCall) operand1, pkAlias, fkAlias);
+ : resolveComputedColumnRef((SqlCall) operand1);
if (tblColRef0 == null || tblColRef1 == null) {
return null;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index c85c447fe0..6459d29a10 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -18,73 +18,39 @@
package org.apache.kylin.query.util;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import org.apache.calcite.plan.hep.HepRelVertex;
-import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.RelVisitor;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
+
import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
-import org.apache.calcite.util.Util;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinTimeoutException;
import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.TableRef;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.BigQueryThresholdUpdater;
import org.apache.kylin.query.IQueryTransformer;
import org.apache.kylin.query.SlowQueryDetector;
import org.apache.kylin.query.exception.UserStopQueryException;
-import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.security.AccessDeniedException;
-import org.apache.kylin.source.adhocquery.IPushDownConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
public class QueryUtil {
- private QueryUtil() {
- }
-
private static final Logger log = LoggerFactory.getLogger("query");
- public static final String DEFAULT_SCHEMA = "DEFAULT";
public static final ImmutableSet<String> REMOVED_TRANSFORMERS = ImmutableSet.of("ReplaceStringWithVarchar");
private static final Pattern SELECT_PATTERN = Pattern.compile("^select", Pattern.CASE_INSENSITIVE);
private static final Pattern SELECT_STAR_PTN = Pattern.compile("^select\\s+\\*\\p{all}*", Pattern.CASE_INSENSITIVE);
@@ -93,9 +59,22 @@ public class QueryUtil {
private static final String COLON = ":";
private static final String SEMI_COLON = ";";
public static final String JDBC = "jdbc";
+ private static final Map<String, IQueryTransformer> QUERY_TRANSFORMER_MAP = Maps.newConcurrentMap();
+
+ static {
+ String[] classNames = KylinConfig.getInstanceFromEnv().getQueryTransformers();
+ for (String clz : classNames) {
+ try {
+ IQueryTransformer transformer = (IQueryTransformer) ClassUtil.newInstance(clz);
+ QUERY_TRANSFORMER_MAP.put(clz, transformer);
+ } catch (Exception e) {
+ log.error("Failed to init query transformer of the sys-config: {}", clz);
+ }
+ }
+ }
- public static List<IQueryTransformer> queryTransformers = Collections.emptyList();
- public static List<IPushDownConverter> pushDownConverters = Collections.emptyList();
+ private QueryUtil() {
+ }
public static boolean isSelectStatement(String sql) {
String sql1 = sql.toLowerCase(Locale.ROOT);
@@ -113,8 +92,17 @@ public class QueryUtil {
return SELECT_STAR_PTN.matcher(sql).find();
}
+ /**
+ * Remove comment from query statement. There are two kind of comment patterns:<br/>
+ * 1. single line comment begins with "--",
+ * with <strong>-- CubePriority(m1,m2)</strong> excluded.<br/>
+ * 2. block comment like "/* comment content *⁄,
+ * with <strong>/*+ MODEL_PRIORITY(m1,m2) *⁄</strong> excluded.<br/>
+ *
+ * @param sql the sql to handle
+ * @return sql without comment
+ */
public static String removeCommentInSql(String sql) {
- // match two patterns, one is "-- comment", the other is "/* comment */"
try {
return new RawSqlParser(sql).parse().getStatementString();
} catch (Exception ex) {
@@ -190,207 +178,9 @@ public class QueryUtil {
return limitMatcher.find() ? originString : replacedString.concat(" limit 1");
}
- public static String massageExpression(NDataModel model, String project, String expression,
- QueryContext.AclInfo aclInfo, boolean isForPushDown) {
- String tempConst = "'" + RandomUtil.randomUUIDStr() + "'";
- StringBuilder forCC = new StringBuilder();
- forCC.append("select ").append(expression).append(" ,").append(tempConst) //
- .append(" FROM ") //
- .append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity());
- appendJoinStatement(model, forCC, false);
-
- String ccSql = KeywordDefaultDirtyHack.transform(forCC.toString());
- try {
- // massage nested CC for drafted model
- Map<String, NDataModel> modelMap = Maps.newHashMap();
- modelMap.put(model.getUuid(), model);
- ccSql = RestoreFromComputedColumn.convertWithGivenModels(ccSql, project, DEFAULT_SCHEMA, modelMap);
- QueryParams queryParams = new QueryParams(project, ccSql, DEFAULT_SCHEMA, false);
- queryParams.setKylinConfig(NProjectManager.getProjectConfig(project));
- queryParams.setAclInfo(aclInfo);
-
- if (isForPushDown) {
- ccSql = massagePushDownSql(queryParams);
- }
- } catch (Exception e) {
- log.warn("Failed to massage SQL expression [{}] with input model {}", ccSql, model.getUuid(), e);
- }
-
- return ccSql.substring("select ".length(), ccSql.indexOf(tempConst) - 2).trim();
- }
-
- public static String massageExpression(NDataModel model, String project, String expression,
- QueryContext.AclInfo aclInfo) {
- return massageExpression(model, project, expression, aclInfo, true);
- }
-
- public static String massageComputedColumn(NDataModel model, String project, ComputedColumnDesc cc,
- QueryContext.AclInfo aclInfo) {
- return massageExpression(model, project, cc.getExpression(), aclInfo);
- }
-
- public static void appendJoinStatement(NDataModel model, StringBuilder sql, boolean singleLine) {
- final String sep = singleLine ? " " : "\n";
- Set<TableRef> dimTableCache = Sets.newHashSet();
- sql.append(sep);
- for (JoinTableDesc lookupDesc : model.getJoinTables()) {
- JoinDesc join = lookupDesc.getJoin();
- TableRef dimTable = lookupDesc.getTableRef();
- if (join == null || StringUtils.isEmpty(join.getType()) || dimTableCache.contains(dimTable)) {
- continue;
- }
-
- TblColRef[] pk = join.getPrimaryKeyColumns();
- TblColRef[] fk = join.getForeignKeyColumns();
- if (pk.length != fk.length) {
- throw new IllegalArgumentException("Invalid join condition of lookup table:" + lookupDesc);
- }
- String joinType = join.getType().toUpperCase(Locale.ROOT);
-
- sql.append(String.format(Locale.ROOT, "%s JOIN \"%s\".\"%s\" as \"%s\"", //
- joinType, dimTable.getTableDesc().getDatabase(), dimTable.getTableDesc().getName(),
- dimTable.getAlias()));
- sql.append(sep);
- sql.append("ON ");
-
- if (pk.length == 0 && join.getNonEquiJoinCondition() != null) {
- sql.append(join.getNonEquiJoinCondition().getExpr());
- dimTableCache.add(dimTable);
- } else {
- String collect = IntStream.range(0, pk.length) //
- .mapToObj(i -> fk[i].getDoubleQuoteExpressionInSourceDB() + " = "
- + pk[i].getDoubleQuoteExpressionInSourceDB())
- .collect(Collectors.joining(" AND ", "", sep));
- sql.append(collect);
- dimTableCache.add(dimTable);
- }
- }
- }
-
- public static SqlSelect extractSqlSelect(SqlCall selectOrOrderby) {
- SqlSelect sqlSelect = null;
-
- if (selectOrOrderby instanceof SqlSelect) {
- sqlSelect = (SqlSelect) selectOrOrderby;
- } else if (selectOrOrderby instanceof SqlOrderBy) {
- SqlOrderBy sqlOrderBy = ((SqlOrderBy) selectOrOrderby);
- if (sqlOrderBy.query instanceof SqlSelect) {
- sqlSelect = (SqlSelect) sqlOrderBy.query;
- }
- }
-
- return sqlSelect;
- }
-
- public static boolean isJoinOnlyOneAggChild(KapJoinRel joinRel) {
- RelNode joinLeftChild;
- RelNode joinRightChild;
- final RelNode joinLeft = joinRel.getLeft();
- final RelNode joinRight = joinRel.getRight();
- if (joinLeft instanceof RelSubset && joinRight instanceof RelSubset) {
- final RelSubset joinLeftChildSub = (RelSubset) joinLeft;
- final RelSubset joinRightChildSub = (RelSubset) joinRight;
- joinLeftChild = Util.first(joinLeftChildSub.getBest(), joinLeftChildSub.getOriginal());
- joinRightChild = Util.first(joinRightChildSub.getBest(), joinRightChildSub.getOriginal());
-
- } else if (joinLeft instanceof HepRelVertex && joinRight instanceof HepRelVertex) {
- joinLeftChild = ((HepRelVertex) joinLeft).getCurrentRel();
- joinRightChild = ((HepRelVertex) joinRight).getCurrentRel();
- } else {
- return false;
- }
-
- String project = QueryContext.current().getProject();
- if (project != null && NProjectManager.getProjectConfig(project).isEnhancedAggPushDownEnabled()
- && RelAggPushDownUtil.canRelAnsweredBySnapshot(project, joinRight)
- && RelAggPushDownUtil.isUnmatchedJoinRel(joinRel)) {
- QueryContext.current().setEnhancedAggPushDown(true);
- return true;
- }
-
- return isContainAggregate(joinLeftChild) ^ isContainAggregate(joinRightChild);
- }
-
- private static boolean isContainAggregate(RelNode node) {
- boolean[] isContainAggregate = new boolean[] { false };
- new RelVisitor() {
- @Override
- public void visit(RelNode node, int ordinal, RelNode parent) {
- if (isContainAggregate[0]) {
- // pruning
- return;
- }
- RelNode relNode = node;
- if (node instanceof RelSubset) {
- relNode = Util.first(((RelSubset) node).getBest(), ((RelSubset) node).getOriginal());
- } else if (node instanceof HepRelVertex) {
- relNode = ((HepRelVertex) node).getCurrentRel();
- }
- if (relNode instanceof Aggregate) {
- isContainAggregate[0] = true;
- }
- super.visit(relNode, ordinal, parent);
- }
- }.go(node);
- return isContainAggregate[0];
- }
-
- public static boolean isCast(RexNode rexNode) {
- if (!(rexNode instanceof RexCall)) {
- return false;
- }
- return SqlKind.CAST == rexNode.getKind();
- }
-
- public static boolean isPlainTableColumn(int colIdx, RelNode relNode) {
- if (relNode instanceof HepRelVertex) {
- relNode = ((HepRelVertex) relNode).getCurrentRel();
- }
- if (relNode instanceof TableScan) {
- return true;
- } else if (relNode instanceof Join) {
- Join join = (Join) relNode;
- int offset = 0;
- for (RelNode input : join.getInputs()) {
- if (colIdx >= offset && colIdx < offset + input.getRowType().getFieldCount()) {
- return isPlainTableColumn(colIdx - offset, input);
- }
- offset += input.getRowType().getFieldCount();
- }
- } else if (relNode instanceof Project) {
- RexNode inputRex = ((Project) relNode).getProjects().get(colIdx);
- if (inputRex instanceof RexInputRef) {
- return isPlainTableColumn(((RexInputRef) inputRex).getIndex(), ((Project) relNode).getInput());
- }
- } else if (relNode instanceof Filter) {
- return isPlainTableColumn(colIdx, relNode.getInput(0));
- }
- return false;
- }
-
- public static boolean containCast(RexNode rexNode) {
- if (!(rexNode instanceof RexCall)) {
- return false;
- }
- if (SqlKind.CAST == rexNode.getKind()) {
- RexNode operand = ((RexCall) rexNode).getOperands().get(0);
- return !(operand instanceof RexCall) || operand.getKind() == SqlKind.CASE;
- }
-
- return false;
- }
-
- public static boolean isNotNullLiteral(RexNode node) {
- return !isNullLiteral(node);
- }
-
- public static boolean isNullLiteral(RexNode node) {
- return node instanceof RexLiteral && ((RexLiteral) node).isNull();
- }
-
public static String massageSql(QueryParams queryParams) {
- String massagedSql = normalMassageSql(queryParams.getKylinConfig(), queryParams.getSql(),
- queryParams.getLimit(), queryParams.getOffset());
+ String massagedSql = appendLimitOffset(queryParams.getProject(), queryParams.getSql(), queryParams.getLimit(),
+ queryParams.getOffset());
queryParams.setSql(massagedSql);
massagedSql = transformSql(queryParams);
QueryContext.current().record("massage");
@@ -404,10 +194,15 @@ public class QueryUtil {
}
private static String transformSql(QueryParams queryParams) {
- // customizable SQL transformation
- initQueryTransformersIfNeeded(queryParams.getKylinConfig(), queryParams.isCCNeeded());
String sql = queryParams.getSql();
- for (IQueryTransformer t : queryTransformers) {
+ String[] classes = queryParams.getKylinConfig().getQueryTransformers();
+ List<IQueryTransformer> transformers = fetchTransformers(queryParams.isCCNeeded(), classes);
+ if (log.isDebugEnabled()) {
+ log.debug("All used query transformers are: {}", transformers.stream()
+ .map(clz -> clz.getClass().getCanonicalName()).collect(Collectors.joining(",")));
+ }
+
+ for (IQueryTransformer t : transformers) {
QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of " + t.getClass(),
"Current step: SQL transformation.");
sql = t.transform(sql, queryParams.getProject(), queryParams.getDefaultSchema());
@@ -415,14 +210,14 @@ public class QueryUtil {
return sql;
}
- private static String trimRightSemiColon(String sql) {
+ public static String trimRightSemiColon(String sql) {
while (sql.endsWith(SEMI_COLON)) {
sql = sql.substring(0, sql.length() - 1).trim();
}
return sql;
}
- public static String normalMassageSql(KylinConfig kylinConfig, String sql, int limit, int offset) {
+ public static String appendLimitOffset(String project, String sql, int limit, int offset) {
sql = sql.trim();
sql = sql.replace("\r", StringUtils.SPACE).replace("\n", System.getProperty("line.separator"));
sql = trimRightSemiColon(sql);
@@ -430,18 +225,19 @@ public class QueryUtil {
//Split keywords and variables from sql by punctuation and whitespace character
List<String> sqlElements = Lists.newArrayList(sql.toLowerCase(Locale.ROOT).split("(?![._'\"`])\\p{P}|\\s+"));
- Integer maxRows = kylinConfig.getMaxResultRows();
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
+ Integer maxRows = projectConfig.getMaxResultRows();
if (maxRows != null && maxRows > 0 && (maxRows < limit || limit <= 0)) {
limit = maxRows;
}
// https://issues.apache.org/jira/browse/KYLIN-2649
- if (kylinConfig.getForceLimit() > 0 && limit <= 0 && !sql.toLowerCase(Locale.ROOT).contains("limit")
+ if (projectConfig.getForceLimit() > 0 && limit <= 0 && !sql.toLowerCase(Locale.ROOT).contains("limit")
&& isSelectStarStatement(sql)) {
- limit = kylinConfig.getForceLimit();
+ limit = projectConfig.getForceLimit();
}
- if (checkBigQueryPushDown(kylinConfig)) {
+ if (isBigQueryPushDownCapable(projectConfig)) {
long bigQueryThreshold = BigQueryThresholdUpdater.getBigQueryThreshold();
if (limit <= 0 && bigQueryThreshold > 0) {
log.info("Big query route to pushdown, Add limit {} to sql.", bigQueryThreshold);
@@ -460,117 +256,51 @@ public class QueryUtil {
return sql;
}
- public static boolean checkBigQueryPushDown(KylinConfig kylinConfig) {
+ public static boolean isBigQueryPushDownCapable(KylinConfig kylinConfig) {
return kylinConfig.isBigQueryPushDown()
&& JDBC.equals(KapConfig.getInstanceFromEnv().getShareStateSwitchImplement());
}
- public static void initQueryTransformersIfNeeded(KylinConfig kylinConfig, boolean isCCNeeded) {
- String[] currentTransformers = queryTransformers.stream().map(Object::getClass).map(Class::getCanonicalName)
- .toArray(String[]::new);
- String[] configTransformers = kylinConfig.getQueryTransformers();
- boolean containsCCTransformer = Arrays.asList(configTransformers)
- .contains("org.apache.kylin.query.util.ConvertToComputedColumn");
- boolean transformersEqual = Objects.deepEquals(currentTransformers, configTransformers);
- if (transformersEqual && (isCCNeeded || !containsCCTransformer)) {
- return;
- }
-
- List<IQueryTransformer> transformers = initTransformers(isCCNeeded, configTransformers);
- queryTransformers = Collections.unmodifiableList(transformers);
- log.debug("SQL transformer: {}", queryTransformers);
- }
-
- public static List<IQueryTransformer> initTransformers(boolean isCCNeeded, String[] configTransformers) {
+ public static List<IQueryTransformer> fetchTransformers(boolean isCCNeeded, String[] configTransformers) {
List<IQueryTransformer> transformers = Lists.newArrayList();
- List<String> classList = Lists.newArrayList(configTransformers);
- classList.removeIf(clazz -> {
- String name = clazz.substring(clazz.lastIndexOf(".") + 1);
- return REMOVED_TRANSFORMERS.contains(name);
- });
-
- for (String clz : classList) {
- if (!isCCNeeded && clz.equals("org.apache.kylin.query.util.ConvertToComputedColumn"))
+ for (String clz : configTransformers) {
+ String name = clz.substring(clz.lastIndexOf('.') + 1);
+ if (REMOVED_TRANSFORMERS.contains(name)) {
continue;
-
- try {
- IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
-
- transformers.add(t);
- } catch (Exception e) {
- throw new IllegalStateException("Failed to init query transformer", e);
}
- }
- return transformers;
- }
-
- // fix KE-34379，filter "/*+ MODEL_PRIORITY({cube_name}) */" hint
- private static final Pattern SQL_HINT_ERASER = Pattern
- .compile("/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/");
-
- public static String massagePushDownSql(QueryParams queryParams) {
- String sql = queryParams.getSql();
- sql = trimRightSemiColon(sql);
-
- sql = SQL_HINT_ERASER.matcher(sql).replaceAll("");
- initPushDownConvertersIfNeeded(queryParams.getKylinConfig());
- for (IPushDownConverter converter : pushDownConverters) {
- QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of " + converter.getClass(),
- "Current step: Massage push-down sql. ");
- sql = converter.convert(sql, queryParams.getProject(), queryParams.getDefaultSchema());
- }
-
- sql = replaceDoubleQuoteToSingle(sql);
- return sql;
- }
-
- // To keep the results of cube and pushDown are same, we need to replace here
- public static String replaceDoubleQuoteToSingle(String originSql) {
- boolean inStrVal = false;
- boolean needTransfer = false;
- char[] res = originSql.toCharArray();
- for (int i = 0; i < res.length; i++) {
- if (res[i] == '\'') {
- if (inStrVal) {
- if (needTransfer) {
- res[i - 1] = '\\';
- needTransfer = false;
- } else {
- needTransfer = true;
- }
- } else {
- inStrVal = true;
- }
+ IQueryTransformer transformer;
+ if (QUERY_TRANSFORMER_MAP.containsKey(clz)) {
+ transformer = QUERY_TRANSFORMER_MAP.get(clz);
} else {
- if (needTransfer) {
- inStrVal = false;
- needTransfer = false;
+ try {
+ transformer = (IQueryTransformer) ClassUtil.newInstance(clz);
+ QUERY_TRANSFORMER_MAP.putIfAbsent(clz, transformer);
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to init query transformer", e);
}
}
+ if (transformer instanceof ConvertToComputedColumn) {
+ if (isCCNeeded) {
+ transformers.add(transformer);
+ }
+ } else {
+ transformers.add(transformer);
+ }
}
- return new String(res);
+ return transformers;
}
- static void initPushDownConvertersIfNeeded(KylinConfig kylinConfig) {
- String[] currentConverters = pushDownConverters.stream().map(Object::getClass).map(Class::getCanonicalName)
- .toArray(String[]::new);
- String[] configConverters = kylinConfig.getPushDownConverterClassNames();
- boolean skipInit = Objects.deepEquals(currentConverters, configConverters);
-
- if (skipInit) {
- return;
- }
-
- List<IPushDownConverter> converters = Lists.newArrayList();
- for (String clz : configConverters) {
- try {
- IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(clz);
- converters.add(converter);
- } catch (Exception e) {
- throw new IllegalStateException("Failed to init pushdown converter", e);
+ public static SqlSelect extractSqlSelect(SqlCall selectOrOrderby) {
+ SqlSelect sqlSelect = null;
+ if (selectOrOrderby instanceof SqlSelect) {
+ sqlSelect = (SqlSelect) selectOrOrderby;
+ } else if (selectOrOrderby instanceof SqlOrderBy) {
+ SqlOrderBy sqlOrderBy = ((SqlOrderBy) selectOrOrderby);
+ if (sqlOrderBy.query instanceof SqlSelect) {
+ sqlSelect = (SqlSelect) sqlOrderBy.query;
}
}
- pushDownConverters = Collections.unmodifiableList(converters);
+ return sqlSelect;
}
public static void checkThreadInterrupted(String errorMsgLog, String stepInfo) {
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
index 2d9e81b01b..2b0eeb31d8 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
@@ -35,6 +35,7 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
import org.apache.kylin.metadata.query.QueryTimesResponse;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
@@ -114,7 +115,7 @@ public class ModelServiceQueryTest extends SourceTestCase {
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = PushDownUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.createAclInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
diff --git a/src/query/pom.xml b/src/query/pom.xml
index ed43542bf9..2c5b3a9a54 100644
--- a/src/query/pom.xml
+++ b/src/query/pom.xml
@@ -20,7 +20,7 @@
<packaging>jar</packaging>
<name>Kylin - Query</name>
<url>http://kylin.apache.org</url>
- <description>Kylin Query Engine</description>
+ <description>Kylin Query Engine</description>
<parent>
<groupId>org.apache.kylin</groupId>
@@ -116,6 +116,13 @@
<scope>provided</scope>
</dependency>
+ <!--test jars-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metadata</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java
index 6a015f6ada..a2bfb3a03b 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java
@@ -18,9 +18,6 @@
package io.kyligence.kap.query.optrule;
-import static org.apache.kylin.query.util.QueryUtil.isCast;
-import static org.apache.kylin.query.util.QueryUtil.isNotNullLiteral;
-
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
@@ -55,6 +52,7 @@ import org.apache.kylin.query.relnode.ContextUtil;
import org.apache.kylin.query.util.AggExpressionUtil;
import org.apache.kylin.query.util.AggExpressionUtil.AggExpression;
import org.apache.kylin.query.util.AggExpressionUtil.GroupExpression;
+import org.apache.kylin.query.util.RuleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +67,7 @@ public abstract class AbstractAggCaseWhenFunctionRule extends RelOptRule {
private static final Logger logger = LoggerFactory.getLogger(AbstractAggCaseWhenFunctionRule.class);
- public AbstractAggCaseWhenFunctionRule(RelOptRuleOperand operand, RelBuilderFactory relBuilderFactory,
+ protected AbstractAggCaseWhenFunctionRule(RelOptRuleOperand operand, RelBuilderFactory relBuilderFactory,
String description) {
super(operand, relBuilderFactory, description);
}
@@ -319,7 +317,7 @@ public abstract class AbstractAggCaseWhenFunctionRule extends RelOptRule {
List<RexNode> values = aggExpression.getValuesList();
for (int i = 0; i < values.size(); i++) {
aggExpression.getBottomAggValuesInput()[i] = bottomProjectList.size();
- if (isCast(values.get(i))) {
+ if (RuleUtils.isCast(values.get(i))) {
RexNode rexNode = ((RexCall) (values.get(i))).operands.get(0);
DataType dataType = DataType.getType(rexNode.getType().getSqlTypeName().getName());
if (!AggExpressionUtil.isSum(aggExpression.getAggCall().getAggregation().kind)
@@ -328,7 +326,7 @@ public abstract class AbstractAggCaseWhenFunctionRule extends RelOptRule {
} else {
bottomProjectList.add(values.get(i));
}
- } else if (isNotNullLiteral(values.get(i))) {
+ } else if (RuleUtils.isNotNullLiteral(values.get(i))) {
bottomProjectList.add(values.get(i));
} else {
bottomProjectList.add(rexBuilder.makeBigintLiteral(BigDecimal.ZERO));
@@ -405,7 +403,8 @@ public abstract class AbstractAggCaseWhenFunctionRule extends RelOptRule {
thenNode = relBuilder.getRexBuilder().makeCast(((RexCall) thenNode).type,
relBuilder.getRexBuilder().makeInputRef(relBuilder.peek(),
aggExpression.getTopProjValuesInput()[whenIndex]));
- } else if (isNotNullLiteral(thenNode)) {// TODO? keep null or sum(null)
+ } else if (RuleUtils.isNotNullLiteral(thenNode)) {
+ // keep null or sum(null)?
thenNode = relBuilder.getRexBuilder().makeInputRef(relBuilder.peek(),
aggExpression.getTopProjValuesInput()[whenIndex]);
}
@@ -415,7 +414,8 @@ public abstract class AbstractAggCaseWhenFunctionRule extends RelOptRule {
if (isNeedTackCast(elseNode)) {
elseNode = relBuilder.getRexBuilder().makeCast(((RexCall) elseNode).type, relBuilder.getRexBuilder()
.makeInputRef(relBuilder.peek(), aggExpression.getTopProjValuesInput()[whenIndex]));
- } else if (isNotNullLiteral(elseNode)) {// TODO? keep null or sum(null)
+ } else if (RuleUtils.isNotNullLiteral(elseNode)) {
+ // keep null or sum(null)?
elseNode = relBuilder.getRexBuilder().makeInputRef(relBuilder.peek(),
aggExpression.getTopProjValuesInput()[whenIndex]);
}
@@ -467,7 +467,7 @@ public abstract class AbstractAggCaseWhenFunctionRule extends RelOptRule {
}
protected boolean isNeedTackCast(RexNode rexNode) {
- return isCast(rexNode);
+ return RuleUtils.isCast(rexNode);
}
private static final String BOTTOM_AGG_PREFIX = "SUB_AGG$";
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java
index ddf0e6d303..46ee6b609e 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java
@@ -18,10 +18,6 @@
package io.kyligence.kap.query.optrule;
-import static org.apache.kylin.query.util.QueryUtil.isCast;
-import static org.apache.kylin.query.util.QueryUtil.isNullLiteral;
-import static org.apache.kylin.query.util.QueryUtil.isPlainTableColumn;
-
import java.util.ArrayList;
import java.util.List;
@@ -62,6 +58,7 @@ import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapProjectRel;
import org.apache.kylin.query.util.AggExpressionUtil;
import org.apache.kylin.query.util.AggExpressionUtil.AggExpression;
+import org.apache.kylin.query.util.RuleUtils;
/**
* COUNT(DISTINCT (CASE WHEN ... THEN COLUMN ELSE NULL))
@@ -119,11 +116,11 @@ public class CountDistinctCaseWhenFunctionRule extends AbstractAggCaseWhenFuncti
}
private boolean isSimpleCaseWhen(Project inputProject, RexNode n1, RexNode n2) {
- if (isNullLiteral(n1)) {
+ if (RuleUtils.isNullLiteral(n1)) {
if (n2 instanceof RexInputRef) {
- return isPlainTableColumn(((RexInputRef) n2).getIndex(), inputProject.getInput(0));
- } else if (isCast(n2) && ((RexCall) n2).getOperands().get(0) instanceof RexInputRef) {
- return isPlainTableColumn(((RexInputRef) ((RexCall) n2).getOperands().get(0)).getIndex(),
+ return RuleUtils.isPlainTableColumn(((RexInputRef) n2).getIndex(), inputProject.getInput(0));
+ } else if (RuleUtils.isCast(n2) && ((RexCall) n2).getOperands().get(0) instanceof RexInputRef) {
+ return RuleUtils.isPlainTableColumn(((RexInputRef) ((RexCall) n2).getOperands().get(0)).getIndex(),
inputProject.getInput(0)) && !isNeedTackCast(n2);
}
}
@@ -209,7 +206,7 @@ public class CountDistinctCaseWhenFunctionRule extends AbstractAggCaseWhenFuncti
@Override
protected boolean isValidAggColumnExpr(RexNode rexNode) {
- return !isNullLiteral(rexNode);
+ return !RuleUtils.isNullLiteral(rexNode);
}
/**
@@ -217,7 +214,7 @@ public class CountDistinctCaseWhenFunctionRule extends AbstractAggCaseWhenFuncti
*/
@Override
protected boolean isNeedTackCast(RexNode rexNode) {
- if (!isCast(rexNode)) {
+ if (!RuleUtils.isCast(rexNode)) {
return false;
}
return !SqlTypeUtil.canCastFrom(rexNode.getType(), ((RexCall) rexNode).getOperands().get(0).getType(), false);
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java
index e6d8ca19cd..984ec00f4e 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java
@@ -40,7 +40,7 @@ import org.apache.calcite.util.mapping.Mappings;
import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.RuleUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -71,7 +71,7 @@ public class KapAggFilterTransposeRule extends RelOptRule {
final KapJoinRel joinRel = call.rel(2);
//Only one agg child of join is accepted
- return QueryUtil.isJoinOnlyOneAggChild(joinRel);
+ return RuleUtils.isJoinOnlyOneAggChild(joinRel);
}
@Override
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java
index 71bd01daa3..b1a26b4b76 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java
@@ -57,7 +57,7 @@ import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.Mappings;
import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapJoinRel;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.RuleUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -89,7 +89,7 @@ public class KapAggJoinTransposeRule extends RelOptRule {
final KapAggregateRel aggregate = call.rel(0);
final KapJoinRel joinRel = call.rel(1);
//Only one agg child of join is accepted
- return !aggregate.isContainCountDistinct() && QueryUtil.isJoinOnlyOneAggChild(joinRel);
+ return !aggregate.isContainCountDistinct() && RuleUtils.isJoinOnlyOneAggChild(joinRel);
}
@Override
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java
index d4ba23a7b7..3b532d778b 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java
@@ -39,7 +39,7 @@ import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.relnode.KapProjectRel;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.RuleUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -72,7 +72,7 @@ public class KapAggProjectMergeRule extends RelOptRule {
? call.rel(3)
: call.rel(2);
//Only one agg child of join is accepted
- if (!QueryUtil.isJoinOnlyOneAggChild(joinRel)) {
+ if (!RuleUtils.isJoinOnlyOneAggChild(joinRel)) {
return false;
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java
index c35e7b2ff5..6c2d7f107f 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java
@@ -48,7 +48,7 @@ import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.relnode.KapProjectRel;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.RuleUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -93,7 +93,7 @@ public class KapAggProjectTransposeRule extends RelOptRule {
}
//Only one agg child of join is accepted
- if (!QueryUtil.isJoinOnlyOneAggChild(joinRel)) {
+ if (!RuleUtils.isJoinOnlyOneAggChild(joinRel)) {
return false;
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java
index 01addae49b..16f62d12d0 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java
@@ -35,7 +35,7 @@ import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.relnode.KapProjectRel;
import org.apache.kylin.query.relnode.KapRel;
-import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.RuleUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -68,7 +68,7 @@ public class KapCountDistinctJoinRule extends RelOptRule {
} else {
join = call.rel(2);
}
- return aggregate.isContainCountDistinct() && QueryUtil.isJoinOnlyOneAggChild(join);
+ return aggregate.isContainCountDistinct() && RuleUtils.isJoinOnlyOneAggChild(join);
}
@Override
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java
index f9608854d7..53e568de1c 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java
@@ -18,9 +18,6 @@
package io.kyligence.kap.query.optrule;
-import static org.apache.kylin.query.util.QueryUtil.containCast;
-import static org.apache.kylin.query.util.QueryUtil.isNotNullLiteral;
-
import java.math.BigDecimal;
import java.util.List;
import java.util.stream.Collectors;
@@ -53,6 +50,7 @@ import org.apache.kylin.query.relnode.ContextUtil;
import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapProjectRel;
import org.apache.kylin.query.util.AggExpressionUtil;
+import org.apache.kylin.query.util.RuleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,7 +75,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
}
List<RexNode> childExps = project.getChildExps();
for (RexNode rexNode : childExps) {
- if (containCast(rexNode)) {
+ if (RuleUtils.containCast(rexNode)) {
return true;
}
}
@@ -93,7 +91,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
if (AggExpressionUtil.isSum(aggCall.getAggregation().kind)) {
int index = aggCall.getArgList().get(0);
RexNode value = originalProject.getProjects().get(index);
- if (containCast(value)) {
+ if (RuleUtils.containCast(value)) {
RexNode rexNode = ((RexCall) value).getOperands().get(0);
DataType dataType = DataType.getType(rexNode.getType().getSqlTypeName().getName());
return dataType.isNumberFamily() || dataType.isIntegerFamily();
@@ -123,7 +121,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
relBuilder.push(oldProject.getInput());
List<AggExpressionUtil.AggExpression> aggExpressions = oldAgg.getAggCallList().stream()
- .map(call -> new AggExpressionUtil.AggExpression(call)).collect(Collectors.toList());
+ .map(AggExpressionUtil.AggExpression::new).collect(Collectors.toList());
// #1 Build bottom project
List<RexNode> bottomProjectList = buildBottomProject(oldProject, aggExpressions);
@@ -140,8 +138,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
List<RexNode> caseProjList = buildTopProject(relBuilder, oldProject, oldAgg, aggExpressions);
relBuilder.project(caseProjList);
- RelNode relNode = relBuilder.build();
- return relNode;
+ return relBuilder.build();
}
private List<RexNode> buildBottomProject(Project oldProject, List<AggExpressionUtil.AggExpression> aggExpressions) {
@@ -156,7 +153,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
if (AggExpressionUtil.isSum(aggCall.getAggregation().kind)) {
int index = aggCall.getArgList().get(0);
RexNode value = oldProject.getProjects().get(index);
- if (containCast(value)) {
+ if (RuleUtils.containCast(value)) {
bottomProjectList.set(index, ((RexCall) (value)).operands.get(0));
RelDataType type = ((RexCall) (value)).operands.get(0).getType();
if (type instanceof BasicSqlType && SqlTypeName.INTEGER == type.getSqlTypeName()) {
@@ -205,7 +202,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
if (AggExpressionUtil.isSum(aggCall.getAggregation().kind)) {
int index = aggCall.getArgList().get(0);
RexNode value = oldProject.getProjects().get(index);
- if (containCast(value)) {
+ if (RuleUtils.containCast(value)) {
RelDataType type = ((RexCall) value).type;
if (type instanceof BasicSqlType && type.getPrecision() < aggCall.getType().getPrecision()) {
type = aggCall.getType();
@@ -213,7 +210,7 @@ public class KapSumCastTransposeRule extends RelOptRule {
value = relBuilder.getRexBuilder().makeCast(type,
relBuilder.getRexBuilder().makeInputRef(relBuilder.peek(), i));
topProjectList.add(value);
- } else if (isNotNullLiteral(value)) {
+ } else if (RuleUtils.isNotNullLiteral(value)) {
value = relBuilder.getRexBuilder().makeInputRef(relBuilder.peek(), i);
topProjectList.add(value);
} else {
diff --git a/src/query/src/main/java/org/apache/kylin/query/QueryCli.java b/src/query/src/main/java/org/apache/kylin/query/QueryCli.java
deleted file mode 100644
index b042875653..0000000000
--- a/src/query/src/main/java/org/apache/kylin/query/QueryCli.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.Statement;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.DBUtils;
-
-public class QueryCli {
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_METADATA = OptionBuilder.withArgName("metadata url").hasArg().isRequired()
- .withDescription("Metadata URL").create("metadata");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_SQL = OptionBuilder.withArgName("input sql").hasArg().isRequired()
- .withDescription("SQL").create("sql");
-
- public static void main(String[] args) throws Exception {
-
- Options options = new Options();
- options.addOption(OPTION_METADATA);
- options.addOption(OPTION_SQL);
-
- CommandLineParser parser = new GnuParser();
- CommandLine commandLine = parser.parse(options, args);
- KylinConfig config = KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt()));
- String sql = commandLine.getOptionValue(OPTION_SQL.getOpt());
-
- Connection conn = null;
- Statement stmt = null;
- ResultSet rs = null;
- try {
-
- // remove since this class looks deprecated
- // stmt = conn.createStatement();
- // rs = stmt.executeQuery(sql);
- // int n = 0;
- // ResultSetMetaData meta = rs.getMetaData();
- // while (rs.next()) {
- // n++;
- // for (int i = 1; i <= meta.getColumnCount(); i++) {
- // System.out.println(n + " - " + meta.getColumnLabel(i) + ":\t" + rs.getObject(i));
- // }
- // }
- } finally {
- DBUtils.closeQuietly(rs);
- DBUtils.closeQuietly(stmt);
- DBUtils.closeQuietly(conn);
- }
-
- }
-}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index 81439609e9..cad6d87c86 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -76,8 +76,7 @@ public class QueryRoutingEngine {
public QueryResult queryWithSqlMassage(QueryParams queryParams) throws Exception {
QueryContext.current().setAclInfo(queryParams.getAclInfo());
- KylinConfig projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(queryParams.getProject()).getConfig();
+ KylinConfig projectKylinConfig = NProjectManager.getProjectConfig(queryParams.getProject());
QueryExec queryExec = new QueryExec(queryParams.getProject(), projectKylinConfig, true);
queryParams.setDefaultSchema(queryExec.getDefaultSchemaName());
@@ -230,7 +229,7 @@ public class QueryRoutingEngine {
private boolean checkBigQueryPushDown(QueryParams queryParams) {
KylinConfig kylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
.getProject(queryParams.getProject()).getConfig();
- boolean isPush = QueryUtil.checkBigQueryPushDown(kylinConfig);
+ boolean isPush = QueryUtil.isBigQueryPushDownCapable(kylinConfig);
if (isPush) {
logger.info("Big query route to pushdown.");
}
@@ -271,22 +270,21 @@ public class QueryRoutingEngine {
sqlString = QueryUtil.addLimit(sqlString);
}
- String massagedSql = QueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), sqlString,
- queryParams.getLimit(), queryParams.getOffset());
+ String massagedSql = QueryUtil.appendLimitOffset(queryParams.getProject(), sqlString, queryParams.getLimit(),
+ queryParams.getOffset());
if (isPrepareStatementWithParams(queryParams)) {
QueryContext.current().getMetrics().setCorrectedSql(massagedSql);
}
queryParams.setSql(massagedSql);
queryParams.setSqlException(sqlException);
queryParams.setPrepare(isPrepare);
- return PushDownUtil.tryPushDownQueryToIterator(queryParams);
+ return PushDownUtil.tryIterQuery(queryParams);
}
private boolean isPrepareStatementWithParams(QueryParams queryParams) {
- KylinConfig kylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(queryParams.getProject()).getConfig();
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(queryParams.getProject());
return (KapConfig.getInstanceFromEnv().enablePushdownPrepareStatementWithParams()
- || kylinConfig.enableReplaceDynamicParams()) && queryParams.isPrepareStatementWithParams();
+ || projectConfig.enableReplaceDynamicParams()) && queryParams.isPrepareStatementWithParams();
}
private QueryResult prepareOnly(String correctedSql, QueryExec queryExec, List<List<String>> results,
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/RuleUtils.java b/src/query/src/main/java/org/apache/kylin/query/util/RuleUtils.java
new file mode 100644
index 0000000000..61e9607a85
--- /dev/null
+++ b/src/query/src/main/java/org/apache/kylin/query/util/RuleUtils.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.util;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelVisitor;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.util.Util;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.query.relnode.KapJoinRel;
+
+public class RuleUtils {
+
+ private RuleUtils() {
+ }
+
+ public static boolean isJoinOnlyOneAggChild(KapJoinRel joinRel) {
+ RelNode joinLeftChild;
+ RelNode joinRightChild;
+ final RelNode joinLeft = joinRel.getLeft();
+ final RelNode joinRight = joinRel.getRight();
+ if (joinLeft instanceof RelSubset && joinRight instanceof RelSubset) {
+ final RelSubset joinLeftChildSub = (RelSubset) joinLeft;
+ final RelSubset joinRightChildSub = (RelSubset) joinRight;
+ joinLeftChild = Util.first(joinLeftChildSub.getBest(), joinLeftChildSub.getOriginal());
+ joinRightChild = Util.first(joinRightChildSub.getBest(), joinRightChildSub.getOriginal());
+
+ } else if (joinLeft instanceof HepRelVertex && joinRight instanceof HepRelVertex) {
+ joinLeftChild = ((HepRelVertex) joinLeft).getCurrentRel();
+ joinRightChild = ((HepRelVertex) joinRight).getCurrentRel();
+ } else {
+ return false;
+ }
+
+ String project = QueryContext.current().getProject();
+ if (project != null && NProjectManager.getProjectConfig(project).isEnhancedAggPushDownEnabled()
+ && RelAggPushDownUtil.canRelAnsweredBySnapshot(project, joinRight)
+ && RelAggPushDownUtil.isUnmatchedJoinRel(joinRel)) {
+ QueryContext.current().setEnhancedAggPushDown(true);
+ return true;
+ }
+
+ return isContainAggregate(joinLeftChild) ^ isContainAggregate(joinRightChild);
+ }
+
+ private static boolean isContainAggregate(RelNode node) {
+ boolean[] isContainAggregate = new boolean[] { false };
+ new RelVisitor() {
+ @Override
+ public void visit(RelNode node, int ordinal, RelNode parent) {
+ if (isContainAggregate[0]) {
+ // pruning
+ return;
+ }
+ RelNode relNode = node;
+ if (node instanceof RelSubset) {
+ relNode = Util.first(((RelSubset) node).getBest(), ((RelSubset) node).getOriginal());
+ } else if (node instanceof HepRelVertex) {
+ relNode = ((HepRelVertex) node).getCurrentRel();
+ }
+ if (relNode instanceof Aggregate) {
+ isContainAggregate[0] = true;
+ }
+ super.visit(relNode, ordinal, parent);
+ }
+ }.go(node);
+ return isContainAggregate[0];
+ }
+
+ public static boolean isCast(RexNode rexNode) {
+ if (!(rexNode instanceof RexCall)) {
+ return false;
+ }
+ return SqlKind.CAST == rexNode.getKind();
+ }
+
+ public static boolean isPlainTableColumn(int colIdx, RelNode relNode) {
+ if (relNode instanceof HepRelVertex) {
+ relNode = ((HepRelVertex) relNode).getCurrentRel();
+ }
+ if (relNode instanceof TableScan) {
+ return true;
+ } else if (relNode instanceof Join) {
+ Join join = (Join) relNode;
+ int offset = 0;
+ for (RelNode input : join.getInputs()) {
+ if (colIdx >= offset && colIdx < offset + input.getRowType().getFieldCount()) {
+ return isPlainTableColumn(colIdx - offset, input);
+ }
+ offset += input.getRowType().getFieldCount();
+ }
+ } else if (relNode instanceof Project) {
+ RexNode inputRex = ((Project) relNode).getProjects().get(colIdx);
+ if (inputRex instanceof RexInputRef) {
+ return isPlainTableColumn(((RexInputRef) inputRex).getIndex(), ((Project) relNode).getInput());
+ }
+ } else if (relNode instanceof Filter) {
+ return isPlainTableColumn(colIdx, relNode.getInput(0));
+ }
+ return false;
+ }
+
+ public static boolean containCast(RexNode rexNode) {
+ if (!(rexNode instanceof RexCall)) {
+ return false;
+ }
+ if (SqlKind.CAST == rexNode.getKind()) {
+ RexNode operand = ((RexCall) rexNode).getOperands().get(0);
+ return !(operand instanceof RexCall) || operand.getKind() == SqlKind.CASE;
+ }
+
+ return false;
+ }
+
+ public static boolean isNotNullLiteral(RexNode node) {
+ return !isNullLiteral(node);
+ }
+
+ public static boolean isNullLiteral(RexNode node) {
+ return node instanceof RexLiteral && ((RexLiteral) node).isNull();
+ }
+}
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index 798da7d042..0a57502dcf 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -18,8 +18,10 @@
package org.apache.kylin.query.util;
+import java.util.Properties;
+
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -38,13 +40,13 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testTryPushDownQuery_ForcedToPushDownBasic() throws Exception {
+ public void testTryForcePushDown() {
try {
QueryParams queryParams = new QueryParams();
queryParams.setProject("default");
queryParams.setSelect(true);
queryParams.setForcedToPushDown(true);
- PushDownUtil.tryPushDownQuery(queryParams);
+ PushDownUtil.tryIterQuery(queryParams);
Assert.fail();
} catch (Exception e) {
Assert.assertFalse(e instanceof IllegalArgumentException);
@@ -52,14 +54,14 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testTryPushDownQuery_PushDownDisable() throws Exception {
+ public void testTryWithPushDownDisable() {
try {
overwriteSystemProp("kylin.query.pushdown-enabled", "false");
QueryParams queryParams = new QueryParams();
queryParams.setProject("default");
queryParams.setSelect(true);
queryParams.setForcedToPushDown(true);
- PushDownUtil.tryPushDownQuery(queryParams);
+ PushDownUtil.tryIterQuery(queryParams);
Assert.fail();
} catch (Exception e) {
Assert.assertFalse(e instanceof IllegalArgumentException);
@@ -67,12 +69,68 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testAddBackticksForIdentity() {
+ public void testBacktickQuote() {
String table = "db.table";
- String partitionColumn = "table.column";
- Pair<String, String> backticksIdentityPair = PushDownUtil.addBackTickForIdentity(table, partitionColumn);
- Assert.assertEquals("`db`.`table`", backticksIdentityPair.getFirst());
- Assert.assertEquals("`table`.`column`", backticksIdentityPair.getSecond());
+ Assert.assertEquals("`db`.`table`", String.join(".", PushDownUtil.backtickQuote(table.split("\\."))));
+ }
+
+ @Test
+ public void testMassagePushDownSql() {
+ KylinConfig config = KylinConfig.createKylinConfig(new Properties());
+ try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
+ config.setProperty("kylin.query.pushdown.converter-class-names",
+ SparkSQLFunctionConverter.class.getCanonicalName());
+ String sql = "SELECT \"Z_PROVDASH_UM_ED\".\"GENDER\" AS \"GENDER\",\n"
+ + "SUM({fn CONVERT(0, SQL_BIGINT)}) AS \"sum_Calculation_336925569152049156_ok\"\n"
+ + "FROM \"POPHEALTH_ANALYTICS\".\"Z_PROVDASH_UM_ED\" \"Z_PROVDASH_UM_ED\"";
+
+ QueryParams queryParams = new QueryParams("", sql, "default", false);
+ queryParams.setKylinConfig(config);
+ String massagedSql = PushDownUtil.massagePushDownSql(queryParams);
+ String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`,\n"
+ + "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
+ + "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` `Z_PROVDASH_UM_ED`";
+ Assert.assertEquals(expectedSql, massagedSql);
+ }
+ }
+
+ @Test
+ public void testMassagePushDownSqlWithDoubleQuote() {
+ KylinConfig config = KylinConfig.createKylinConfig(new Properties());
+ String sql = "select '''',trans_id from test_kylin_fact where LSTG_FORMAT_NAME like '%''%' group by trans_id limit 2;";
+ QueryParams queryParams = new QueryParams("", sql, "default", false);
+ queryParams.setKylinConfig(config);
+ String massagedSql = PushDownUtil.massagePushDownSql(queryParams);
+ String expectedSql = "select '\\'', `TRANS_ID` from `TEST_KYLIN_FACT` where `LSTG_FORMAT_NAME` like '%\\'%' group by `TRANS_ID` limit 2";
+ Assert.assertEquals(expectedSql, massagedSql);
+ }
+
+ @Test
+ public void testMassagePushDownSqlWithDialectConverter() {
+ KylinConfig config = KylinConfig.createKylinConfig(new Properties());
+ try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
+ config.setProperty("kylin.query.pushdown.converter-class-names",
+ "org.apache.kylin.query.util.DialectConverter,org.apache.kylin.source.adhocquery.DoubleQuotePushDownConverter,"
+ + SparkSQLFunctionConverter.class.getCanonicalName());
+ String sql = "SELECT \"Z_PROVDASH_UM_ED\".\"GENDER\" AS \"GENDER\",\n"
+ + "SUM({fn CONVERT(0, SQL_BIGINT)}) AS \"sum_Calculation_336925569152049156_ok\"\n"
+ + "FROM \"POPHEALTH_ANALYTICS\".\"Z_PROVDASH_UM_ED\" \"Z_PROVDASH_UM_ED\""
+ + " fetch first 1 rows only";
+
+ QueryParams queryParams = new QueryParams("", sql, "default", false);
+ queryParams.setKylinConfig(config);
+ String massagedSql = PushDownUtil.massagePushDownSql(queryParams);
+ String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`, "
+ + "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
+ + "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` AS `Z_PROVDASH_UM_ED`\n" + "LIMIT 1";
+ Assert.assertEquals(expectedSql, massagedSql);
+ }
}
-}
\ No newline at end of file
+ @Test
+ public void testReplaceDoubleQuoteToSingle() {
+ String sql = "select ab from table where aa = '' and bb = '''as''n'''";
+ String resSql = "select ab from table where aa = '' and bb = '\\'as\\'n\\''";
+ Assert.assertEquals(resSql, PushDownUtil.replaceEscapedQuote(sql));
+ }
+}
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index 3d0ac0f1c5..0b2120263f 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -19,33 +19,31 @@
package org.apache.kylin.query.util;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.when;
import java.sql.SQLException;
-import java.util.Collections;
+import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.query.IQueryTransformer;
import org.apache.kylin.query.security.AccessDeniedException;
+import org.apache.kylin.util.MetadataTestUtils;
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.MockedStatic;
+
+import com.google.common.collect.Maps;
public class QueryUtilTest extends NLocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
- QueryUtil.queryTransformers = Collections.emptyList();
- QueryUtil.pushDownConverters = Collections.emptyList();
this.createTestMetadata();
}
@@ -54,48 +52,40 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
this.cleanupTestMetadata();
}
- public static final String SQL = "select * from table1";
-
@Test
public void testMaxResultRowsEnabled() {
- try (MockedStatic<KylinConfig> kylinConfigMockedStatic = mockStatic(KylinConfig.class)) {
- KylinConfig kylinConfig = mock(KylinConfig.class);
- kylinConfigMockedStatic.when(KylinConfig::getInstanceFromEnv).thenReturn(kylinConfig);
- when(kylinConfig.getMaxResultRows()).thenReturn(15);
- when(kylinConfig.getForceLimit()).thenReturn(14);
- String result = QueryUtil.normalMassageSql(kylinConfig, SQL, 16, 0);
- assertEquals("select * from table1" + "\n" + "LIMIT 15", result);
- }
+ Map<String, String> map = Maps.newHashMap();
+ map.put("kylin.query.max-result-rows", "15");
+ map.put("kylin.query.force-limit", "14");
+ MetadataTestUtils.updateProjectConfig("default", map);
+ String result = QueryUtil.appendLimitOffset("default", "select * from table1", 16, 0);
+ assertEquals("select * from table1" + "\n" + "LIMIT 15", result);
}
@Test
public void testCompareMaxResultRowsAndLimit() {
- try (MockedStatic<KylinConfig> kylinConfigMockedStatic = mockStatic(KylinConfig.class)) {
- KylinConfig kylinConfig = mock(KylinConfig.class);
- kylinConfigMockedStatic.when(KylinConfig::getInstanceFromEnv).thenReturn(kylinConfig);
- when(kylinConfig.getMaxResultRows()).thenReturn(15);
- when(kylinConfig.getForceLimit()).thenReturn(14);
- String result = QueryUtil.normalMassageSql(kylinConfig, SQL, 13, 0);
- assertEquals("select * from table1" + "\n" + "LIMIT 13", result);
- }
+ Map<String, String> map = Maps.newHashMap();
+ map.put("kylin.query.max-result-rows", "15");
+ map.put("kylin.query.force-limit", "14");
+ MetadataTestUtils.updateProjectConfig("default", map);
+ String result = QueryUtil.appendLimitOffset("default", "select * from table1", 13, 0);
+ assertEquals("select * from table1" + "\n" + "LIMIT 13", result);
}
@Test
public void testMassageSql() {
- KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
- config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName());
-
- String sql = "SELECT * FROM TABLE";
- QueryParams queryParams1 = new QueryParams(config, sql, "", 100, 20, "", true);
- String newSql = QueryUtil.massageSql(queryParams1);
- Assert.assertEquals("SELECT * FROM TABLE\nLIMIT 100\nOFFSET 20", newSql);
-
- String sql2 = "SELECT SUM({fn convert(0, INT)}) from TABLE";
- QueryParams queryParams2 = new QueryParams(config, sql2, "", 0, 0, "", true);
- String newSql2 = QueryUtil.massageSql(queryParams2);
- Assert.assertEquals("SELECT SUM({fn convert(0, INT)}) from TABLE", newSql2);
- }
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName());
+
+ String sql = "SELECT * FROM TABLE1";
+ QueryParams queryParams1 = new QueryParams(config, sql, "default", 100, 20, "", true);
+ String newSql = QueryUtil.massageSql(queryParams1);
+ Assert.assertEquals("SELECT * FROM TABLE1\nLIMIT 100\nOFFSET 20", newSql);
+
+ String sql2 = "SELECT SUM({fn convert(0, INT)}) from TABLE1";
+ QueryParams queryParams2 = new QueryParams(config, sql2, "default", 0, 0, "", true);
+ String newSql2 = QueryUtil.massageSql(queryParams2);
+ Assert.assertEquals("SELECT SUM({fn convert(0, INT)}) from TABLE1", newSql2);
}
@Test
@@ -121,29 +111,27 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
public void testMassageWithoutConvertToComputedColumn() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
- // enable ConvertToComputedColumn
- config.setProperty("kylin.query.transformers", "org.apache.kylin.query.util.ConvertToComputedColumn");
- QueryParams queryParams1 = new QueryParams(config, "SELECT price * item_count FROM test_kylin_fact",
- "default", 0, 0, "DEFAULT", true);
- String newSql1 = QueryUtil.massageSql(queryParams1);
- Assert.assertEquals("SELECT TEST_KYLIN_FACT.DEAL_AMOUNT FROM test_kylin_fact", newSql1);
- QueryParams queryParams2 = new QueryParams(config,
- "SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", "default", 0, 0, "DEFAULT", true);
- newSql1 = QueryUtil.massageSql(queryParams2);
- Assert.assertEquals("SELECT TEST_KYLIN_FACT.DEAL_AMOUNT,DEAL_AMOUNT FROM test_kylin_fact", newSql1);
-
- // disable ConvertToComputedColumn
- config.setProperty("kylin.query.transformers", "");
- QueryParams queryParams3 = new QueryParams(config, "SELECT price * item_count FROM test_kylin_fact",
- "default", 0, 0, "DEFAULT", true);
- String newSql2 = QueryUtil.massageSql(queryParams3);
- Assert.assertEquals("SELECT price * item_count FROM test_kylin_fact", newSql2);
- QueryParams queryParams4 = new QueryParams(config,
- "SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", "default", 0, 0, "DEFAULT", false);
- newSql2 = QueryUtil.massageSql(queryParams4);
- Assert.assertEquals("SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", newSql2);
- }
+ // enable ConvertToComputedColumn
+ config.setProperty("kylin.query.transformers", "org.apache.kylin.query.util.ConvertToComputedColumn");
+ QueryParams queryParams1 = new QueryParams(config, "SELECT price * item_count FROM test_kylin_fact", "default",
+ 0, 0, "DEFAULT", true);
+ String newSql1 = QueryUtil.massageSql(queryParams1);
+ Assert.assertEquals("SELECT TEST_KYLIN_FACT.DEAL_AMOUNT FROM test_kylin_fact", newSql1);
+ QueryParams queryParams2 = new QueryParams(config, "SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact",
+ "default", 0, 0, "DEFAULT", true);
+ newSql1 = QueryUtil.massageSql(queryParams2);
+ Assert.assertEquals("SELECT TEST_KYLIN_FACT.DEAL_AMOUNT,DEAL_AMOUNT FROM test_kylin_fact", newSql1);
+
+ // disable ConvertToComputedColumn
+ config.setProperty("kylin.query.transformers", "");
+ QueryParams queryParams3 = new QueryParams(config, "SELECT price * item_count FROM test_kylin_fact", "default",
+ 0, 0, "DEFAULT", true);
+ String newSql2 = QueryUtil.massageSql(queryParams3);
+ Assert.assertEquals("SELECT price * item_count FROM test_kylin_fact", newSql2);
+ QueryParams queryParams4 = new QueryParams(config, "SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact",
+ "default", 0, 0, "DEFAULT", false);
+ newSql2 = QueryUtil.massageSql(queryParams4);
+ Assert.assertEquals("SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", newSql2);
}
@Test
@@ -196,64 +184,31 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testMassagePushDownSql() {
+ public void testInit() {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
- config.setProperty("kylin.query.pushdown.converter-class-names",
- SparkSQLFunctionConverter.class.getCanonicalName());
- String sql = "SELECT \"Z_PROVDASH_UM_ED\".\"GENDER\" AS \"GENDER\",\n"
- + "SUM({fn CONVERT(0, SQL_BIGINT)}) AS \"sum_Calculation_336925569152049156_ok\"\n"
- + "FROM \"POPHEALTH_ANALYTICS\".\"Z_PROVDASH_UM_ED\" \"Z_PROVDASH_UM_ED\"";
-
- QueryParams queryParams = new QueryParams("", sql, "default", false);
- queryParams.setKylinConfig(config);
- String massagedSql = QueryUtil.massagePushDownSql(queryParams);
- String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`,\n"
- + "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
- + "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` `Z_PROVDASH_UM_ED`";
- Assert.assertEquals(expectedSql, massagedSql);
- }
- }
+ config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName());
+ List<IQueryTransformer> transformers = QueryUtil.fetchTransformers(true, config.getQueryTransformers());
+ Assert.assertEquals(1, transformers.size());
+ Assert.assertTrue(transformers.get(0) instanceof DefaultQueryTransformer);
- @Test
- public void testMassagePushDownSqlWithDoubleQuote() {
- KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- String sql = "select '''',trans_id from test_kylin_fact where LSTG_FORMAT_NAME like '%''%' group by trans_id limit 2;";
- QueryParams queryParams = new QueryParams("", sql, "default", false);
- queryParams.setKylinConfig(config);
- String massagedSql = QueryUtil.massagePushDownSql(queryParams);
- String expectedSql = "select '\\'', `TRANS_ID` from `TEST_KYLIN_FACT` where `LSTG_FORMAT_NAME` like '%\\'%' group by `TRANS_ID` limit 2";
- Assert.assertEquals(expectedSql, massagedSql);
- }
+ config.setProperty("kylin.query.transformers", KeywordDefaultDirtyHack.class.getCanonicalName());
+ transformers = QueryUtil.fetchTransformers(true, config.getQueryTransformers());
+ Assert.assertEquals(1, transformers.size());
+ Assert.assertTrue(transformers.get(0) instanceof KeywordDefaultDirtyHack);
+
+ transformers = QueryUtil.fetchTransformers(false, config.getQueryTransformers());
+ Assert.assertEquals(1, transformers.size());
+ Assert.assertTrue(transformers.get(0) instanceof KeywordDefaultDirtyHack);
+
+ config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName() + ","
+ + ConvertToComputedColumn.class.getCanonicalName());
+ transformers = QueryUtil.fetchTransformers(true, config.getQueryTransformers());
+ Assert.assertEquals(2, transformers.size());
+
+ transformers = QueryUtil.fetchTransformers(false, config.getQueryTransformers());
+ Assert.assertEquals(1, transformers.size());
+ Assert.assertTrue(transformers.get(0) instanceof DefaultQueryTransformer);
- @Test
- public void testInit() {
- KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
-
- config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName());
- Assert.assertEquals(0, QueryUtil.queryTransformers.size());
- QueryUtil.initQueryTransformersIfNeeded(config, true);
- Assert.assertEquals(1, QueryUtil.queryTransformers.size());
- Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof DefaultQueryTransformer);
-
- config.setProperty("kylin.query.transformers", KeywordDefaultDirtyHack.class.getCanonicalName());
- QueryUtil.initQueryTransformersIfNeeded(config, true);
- Assert.assertEquals(1, QueryUtil.queryTransformers.size());
- Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof KeywordDefaultDirtyHack);
-
- QueryUtil.initQueryTransformersIfNeeded(config, false);
- Assert.assertEquals(1, QueryUtil.queryTransformers.size());
- Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof KeywordDefaultDirtyHack);
-
- config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName() + ","
- + ConvertToComputedColumn.class.getCanonicalName());
- QueryUtil.initQueryTransformersIfNeeded(config, true);
- Assert.assertEquals(2, QueryUtil.queryTransformers.size());
- QueryUtil.initQueryTransformersIfNeeded(config, false);
- Assert.assertEquals(1, QueryUtil.queryTransformers.size());
- Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof DefaultQueryTransformer);
- }
}
@Test
@@ -466,7 +421,6 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(
"select LINEORDER.CC_CAST_LO_ORDERKEY from lineorder inner join customer on lineorder.lo_custkey = customer.c_custkey",
newSql2);
-
}
@Test
@@ -518,28 +472,6 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
newSql6);
}
- @Test
- public void testMassagePushDownSqlWithDialectConverter() {
- KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
- config.setProperty("kylin.query.pushdown.converter-class-names",
- "org.apache.kylin.query.util.DialectConverter,org.apache.kylin.source.adhocquery.DoubleQuotePushDownConverter,"
- + SparkSQLFunctionConverter.class.getCanonicalName());
- String sql = "SELECT \"Z_PROVDASH_UM_ED\".\"GENDER\" AS \"GENDER\",\n"
- + "SUM({fn CONVERT(0, SQL_BIGINT)}) AS \"sum_Calculation_336925569152049156_ok\"\n"
- + "FROM \"POPHEALTH_ANALYTICS\".\"Z_PROVDASH_UM_ED\" \"Z_PROVDASH_UM_ED\""
- + " fetch first 1 rows only";
-
- QueryParams queryParams = new QueryParams("", sql, "default", false);
- queryParams.setKylinConfig(config);
- String massagedSql = QueryUtil.massagePushDownSql(queryParams);
- String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`, "
- + "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
- + "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` AS `Z_PROVDASH_UM_ED`\n" + "LIMIT 1";
- Assert.assertEquals(expectedSql, massagedSql);
- }
- }
-
@Test
public void testBigQueryPushDown() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -557,6 +489,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
@Test
public void testBigQueryPushDownByParams() {
+ // no limit offset from backend and front-end
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
String sql1 = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams1 = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
@@ -564,77 +497,104 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
newSql1);
+
+ // both limit and offset are 0
String sql = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
String targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
targetSQL);
+
+ // limit 1 from front-end
queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 1",
targetSQL);
- config.setProperty("kylin.query.max-result-rows", "2");
- queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
+ }
+
+ @Test
+ public void testAddLimitOffsetBetweenBigQueryPushDownByParamsAndMaxResultRows() {
+ KylinConfig config = KylinConfig.createKylinConfig(new Properties());
+ // read project config of `kylin.query.max-result-rows`
+ String sql = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
+ MetadataTestUtils.updateProjectConfig("default", "kylin.query.max-result-rows", "2");
+ QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
+ String targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 2",
targetSQL);
+
+ // read project config of `kylin.query.max-result-rows=2` but front-end limit has a high priority
queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 1",
targetSQL);
- config.setProperty("kylin.query.max-result-rows", "-1");
- config.setProperty("kylin.query.force-limit", "3");
- queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals(
- "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
- targetSQL);
- queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals(
- "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
- + "LIMIT 1",
- targetSQL);
- sql1 = "select * from table1";
- queryParams = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals("select * from table1" + "\n" + "LIMIT 3", targetSQL);
- queryParams = new QueryParams(config, sql1, "default", 2, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals("select * from table1" + "\n" + "LIMIT 2", targetSQL);
- sql1 = "select * from table1 limit 4";
- queryParams = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals("select * from table1 limit 4", targetSQL);
- queryParams = new QueryParams(config, sql1, "default", 2, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals("select * from table1 limit 4", targetSQL);
- config.setProperty("kylin.query.force-limit", "-1");
- config.setProperty("kylin.query.share-state-switch-implement", "jdbc");
- queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals(
- "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
- targetSQL);
- config.setProperty("kylin.query.big-query-pushdown", "true");
- queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = QueryUtil.massageSql(queryParams);
- Assert.assertEquals(
- "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
- targetSQL);
}
@Test
- public void testReplaceDoubleQuoteToSingle() {
- String sql = "select ab from table where aa = '' and bb = '''as''n'''";
- String resSql = "select ab from table where aa = '' and bb = '\\'as\\'n\\''";
- Assert.assertEquals(resSql, QueryUtil.replaceDoubleQuoteToSingle(sql));
+ public void testAddLimitOffsetBetweenBigQueryPushDownWithForceLimit() {
+ KylinConfig config = KylinConfig.createKylinConfig(new Properties());
+ String sql = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
+ // compare the priority of two properties, the `kylin.query.max-result-rows` has higher priority if it is bigger than 0
+ {
+ Map<String, String> map = Maps.newHashMap();
+ map.put("kylin.query.max-result-rows", "-1");
+ map.put("kylin.query.force-limit", "3");
+ MetadataTestUtils.updateProjectConfig("default", map);
+ QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
+ String targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals(
+ "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
+ targetSQL);
+
+ // the front-end param has a higher priority
+ queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
+ targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals(
+ "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ + "LIMIT 1",
+ targetSQL);
+
+ String sql1 = "select * from table1";
+ queryParams = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
+ targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals("select * from table1" + "\n" + "LIMIT 3", targetSQL);
+ queryParams = new QueryParams(config, sql1, "default", 2, 0, "DEFAULT", true);
+ targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals("select * from table1" + "\n" + "LIMIT 2", targetSQL);
+ sql1 = "select * from table1 limit 4";
+ queryParams = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
+ targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals("select * from table1 limit 4", targetSQL);
+ queryParams = new QueryParams(config, sql1, "default", 2, 0, "DEFAULT", true);
+ targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals("select * from table1 limit 4", targetSQL);
+ }
+
+ {
+ Map<String, String> map = Maps.newHashMap();
+ map.put("kylin.query.force-limit", "-1");
+ map.put("kylin.query.share-state-switch-implement", "jdbc");
+ MetadataTestUtils.updateProjectConfig("default", map);
+ QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
+ String targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals(
+ "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
+ targetSQL);
+
+ // add one more property `kylin.query.big-query-pushdown`
+ MetadataTestUtils.updateProjectConfig("default", "kylin.query.big-query-pushdown", "true");
+ queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
+ targetSQL = QueryUtil.massageSql(queryParams);
+ Assert.assertEquals(
+ "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
+ targetSQL);
+ }
}
}
diff --git a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java
index 124c76b4fc..a80d39ed5f 100644
--- a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java
+++ b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java
@@ -38,8 +38,8 @@ import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.engine.spark.IndexDataConstructor;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -196,7 +196,7 @@ public class TDVTHiveTest {
private String runWithHive(String sqlStatement) {
QueryParams queryParams = new QueryParams(project, sqlStatement, "default", false);
queryParams.setKylinConfig(NProjectManager.getProjectConfig(project));
- String afterConvert = QueryUtil.massagePushDownSql(queryParams);
+ String afterConvert = PushDownUtil.massagePushDownSql(queryParams);
// Table schema comes from csv and DATABASE.TABLE is not supported.
String sqlForSpark = ExecAndComp.removeDataBaseInSql(afterConvert);
Dataset<Row> plan = ExecAndComp.querySparkSql(sqlForSpark);
diff --git a/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java b/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
index 9ef91ef98d..afe9add1cd 100644
--- a/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
+++ b/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
@@ -39,6 +39,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.query.util.QueryUtil;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.ModelRequest;
@@ -141,7 +142,7 @@ public class ModelServiceWithSecondStorageTest extends NLocalFileMetadataTestCas
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = PushDownUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.createAclInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index db9c31c9b0..cb395e88ba 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -80,7 +80,6 @@ import org.apache.kylin.metadata.view.LogicalView;
import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.query.pushdown.SparkSubmitter;
import org.apache.kylin.query.util.PushDownUtil;
-
import org.apache.spark.SparkConf;
import org.apache.spark.SparkException;
import org.apache.spark.application.NoRetryException;
@@ -97,7 +96,6 @@ import org.apache.spark.sql.catalyst.rules.Rule;
import org.apache.spark.sql.execution.datasource.AlignmentTableStats;
import org.apache.spark.sql.hive.utils.ResourceDetectUtils;
import org.apache.spark.util.Utils;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,7 +104,6 @@ import com.google.common.collect.Maps;
import org.apache.kylin.engine.spark.job.SegmentBuildJob;
import lombok.val;
-
import scala.runtime.AbstractFunction1;
import scala.runtime.BoxedUnit;
@@ -360,7 +357,7 @@ public abstract class SparkApplication implements Application {
}
// Permission exception will not be retried. Simply let the job fail.
- protected void interceptAccessControlException(Throwable e) throws NoRetryException{
+ protected void interceptAccessControlException(Throwable e) throws NoRetryException {
logger.error("Permission denied.", e);
throw new NoRetryException("Permission denied.");
}
@@ -534,28 +531,27 @@ public abstract class SparkApplication implements Application {
return;
}
val modelManager = NDataModelManager.getInstance(config, project);
- NDataModel modelDesc = modelManager.getDataModelDesc(modelId);
- if (checkRangePartitionTableIsExist(modelDesc)) {
+ NDataModel model = modelManager.getDataModelDesc(modelId);
+ if (checkRangePartitionTableIsExist(model)) {
logger.info("Range partitioned tables do not support pushdown, so do not need to perform subsequent logic");
return;
}
- val partitionDesc = modelDesc.getPartitionDesc();
+ val partitionDesc = model.getPartitionDesc();
if (PartitionDesc.isEmptyPartitionDesc(partitionDesc)
- || org.apache.commons.lang.StringUtils.isEmpty(partitionDesc.getPartitionDateFormat()))
+ || StringUtils.isEmpty(partitionDesc.getPartitionDateFormat()))
return;
- if (CatalogTableType.VIEW().name().equals(modelDesc.getRootFactTable().getTableDesc().getTableType()))
+ if (CatalogTableType.VIEW().name().equals(model.getRootFactTable().getTableDesc().getTableType()))
return;
- String partitionColumn = modelDesc.getPartitionDesc().getPartitionDateColumnRef().getExpressionInSourceDB();
+ String partitionColumn = model.getPartitionDesc().getPartitionDateColumnRef().getBackTickExp();
SparkSession sparkSession = atomicSparkSession.get();
try (SparkSubmitter.OverriddenSparkSession ignored = SparkSubmitter.getInstance()
.overrideSparkSession(sparkSession)) {
- String dateString = PushDownUtil.getFormatIfNotExist(modelDesc.getRootFactTableName(), partitionColumn,
- project);
- val sdf = new SimpleDateFormat(modelDesc.getPartitionDesc().getPartitionDateFormat(),
+ String dateString = PushDownUtil.probeColFormat(model.getRootFactTableName(), partitionColumn, project);
+ val sdf = new SimpleDateFormat(model.getPartitionDesc().getPartitionDateFormat(),
Locale.getDefault(Locale.Category.FORMAT));
val date = sdf.parse(dateString);
if (date == null || !dateString.equals(sdf.format(date))) {
@@ -665,9 +661,8 @@ public abstract class SparkApplication implements Application {
LogicalViewManager viewManager = LogicalViewManager.getInstance(config);
if (StringUtils.isNotBlank(dataflowId)) {
- viewManager
- .findLogicalViewsInModel(project, dataflowId)
- .forEach(view -> LogicalViewLoader.loadView(view.getTableName(), true, ss));
+ viewManager.findLogicalViewsInModel(project, dataflowId)
+ .forEach(view -> LogicalViewLoader.loadView(view.getTableName(), true, ss));
}
if (StringUtils.isNotBlank(tableName)) {
LogicalView view = viewManager.findLogicalViewInProject(getProject(), tableName);
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index 4370252206..a72b7c070d 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -53,7 +53,7 @@ import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.engine.spark.merger.MetadataMerger;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.JobStoppedException;
@@ -660,7 +660,7 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
}
// when layout ids not null, set index count
if (StringUtils.isNotBlank(getParam(NBatchConstants.P_LAYOUT_IDS))) {
- int indexCount = StringUtil.splitAndTrim(getParam(NBatchConstants.P_LAYOUT_IDS), ",").length;
+ int indexCount = StringHelper.splitAndTrim(getParam(NBatchConstants.P_LAYOUT_IDS), ",").length;
setParam(NBatchConstants.P_INDEX_COUNT, String.valueOf(indexCount));
}
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
index ecb0852c4d..03ef1ba57d 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
@@ -24,31 +24,22 @@ import java.util.Locale;
import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.TableRef;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.engine.spark.builder.CreateFlatTable;
import org.apache.kylin.engine.spark.job.NSparkCubingUtil;
+import org.apache.kylin.engine.spark.smarter.IndexDependencyParser;
import org.apache.kylin.metadata.model.BadModelException;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.exception.IllegalCCExpressionException;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
-import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.execution.utils.SchemaProcessor;
import org.apache.spark.sql.util.SparderTypeUtil;
import org.springframework.util.CollectionUtils;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -90,7 +81,7 @@ public class ComputedColumnEvalUtil {
}
try {
evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(), nDataModel, 0, computedColumns.size());
- } catch (AnalysisException e) {
+ } catch (Exception e) {
evalDataTypeOfCCInManual(computedColumns, nDataModel, 0, computedColumns.size());
}
}
@@ -100,7 +91,7 @@ public class ComputedColumnEvalUtil {
for (int i = start; i < end; i++) {
try {
evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(), nDataModel, i, i + 1);
- } catch (AnalysisException e) {
+ } catch (Exception e) {
Preconditions.checkNotNull(computedColumns.get(i));
throw new IllegalCCExpressionException(QueryErrorCode.CC_EXPRESSION_ILLEGAL,
String.format(Locale.ROOT, MsgPicker.getMsg().getCheckCCExpression(),
@@ -111,42 +102,23 @@ public class ComputedColumnEvalUtil {
}
private static void evalDataTypeOfCC(List<ComputedColumnDesc> computedColumns, SparkSession ss,
- NDataModel nDataModel, int start, int end) throws AnalysisException {
- val originDf = generateFullFlatTableDF(ss, nDataModel);
+ NDataModel nDataModel, int start, int end) {
+ IndexDependencyParser parser = new IndexDependencyParser(nDataModel);
+ Dataset<Row> originDf = parser.generateFullFlatTableDF(ss, nDataModel);
originDf.persist();
- Dataset<Row> ds = originDf
- .selectExpr(computedColumns.subList(start, end).stream().map(ComputedColumnDesc::getInnerExpression)
- .map(NSparkCubingUtil::convertFromDotWithBackTick).toArray(String[]::new));
+ String[] ccExprArray = computedColumns.subList(start, end).stream() //
+ .map(ComputedColumnDesc::getInnerExpression) //
+ .map(NSparkCubingUtil::convertFromDotWithBackTick).toArray(String[]::new);
+ Dataset<Row> ds = originDf.selectExpr(ccExprArray);
for (int i = start; i < end; i++) {
String dataType = SparderTypeUtil.convertSparkTypeToSqlType(ds.schema().fields()[i - start].dataType());
computedColumns.get(i).setDatatype(dataType);
}
}
- private static Dataset<Row> generateFullFlatTableDF(SparkSession ss, NDataModel model) {
- // root fact table
- val rootDF = generateDatasetOnTable(ss, model.getRootFactTable());
-
- // look up tables
- val joinTableDFMap = Maps.<JoinTableDesc, Dataset<Row>> newLinkedHashMap();
- model.getJoinTables().forEach(
- joinTable -> joinTableDFMap.put(joinTable, generateDatasetOnTable(ss, joinTable.getTableRef())));
-
- return CreateFlatTable.joinFactTableWithLookupTables(rootDF, joinTableDFMap, model, ss);
- }
-
- private static Dataset<Row> generateDatasetOnTable(SparkSession ss, TableRef tableRef) {
- val tableCols = tableRef.getColumns().stream().map(TblColRef::getColumnDesc)
- .filter(col -> !col.isComputedColumn()).toArray(ColumnDesc[]::new);
- val structType = SchemaProcessor.buildSchemaWithRawTable(tableCols);
- val alias = tableRef.getAlias();
- val dataset = ss.createDataFrame(Lists.newArrayList(), structType).alias(alias);
- return CreateFlatTable.changeSchemaToAliasDotName(dataset, alias);
- }
-
public static boolean resolveCCName(ComputedColumnDesc ccDesc, NDataModel dataModel, List<NDataModel> otherModels) {
- // Resolve CC name, Limit MAX_RENAME_CC_TIME retries to avoid infinite loop
- // TODO: what if the dataModel has more than MAX_RENAME_CC_TIME computed columns?
+ // Resolve CC name, Limit MAX_RENAME_CC_TIME retries to avoid infinite loop.
+ // What if the dataModel has more than MAX_RENAME_CC_TIME computed columns?
int retryCount = 0;
while (retryCount < MAX_RENAME_CC_TIME) {
retryCount++;
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
index fb91ce7933..fdb4d87cd2 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
@@ -112,9 +112,9 @@ class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
}
protected def encodeWithCols(ds: Dataset[Row],
- ccCols: Set[TblColRef],
- dictCols: Set[TblColRef],
- encodeCols: Set[TblColRef]): Dataset[Row] = {
+ ccCols: Set[TblColRef],
+ dictCols: Set[TblColRef],
+ encodeCols: Set[TblColRef]): Dataset[Row] = {
val ccDataset = withColumn(ds, ccCols)
if (seg.isDictReady) {
logInfo(s"Skip already built dict, segment: ${seg.getId} of dataflow: ${seg.getDataflow.getId}")
@@ -128,7 +128,7 @@ class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
val matchedCols = selectColumnsInTable(ds, withCols)
var withDs = ds
matchedCols.foreach(m => withDs = withDs.withColumn(convertFromDot(m.getBackTickIdentity),
- expr(convertFromDot(m.getBackTickExpressionInSourceDB))))
+ expr(convertFromDot(m.getBackTickExp))))
withDs
}
@@ -189,8 +189,8 @@ object CreateFlatTable extends LogEx {
}
def generateLookupTableDataset(model: NDataModel,
- cols: Seq[TblColRef],
- ss: SparkSession): mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]] = {
+ cols: Seq[TblColRef],
+ ss: SparkSession): mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]] = {
val lookupTables = mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]]()
model.getJoinTables.asScala.map(
joinDesc => {
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala
index 209361f089..e58a803c62 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala
@@ -18,10 +18,10 @@
package org.apache.kylin.engine.spark.builder
+import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.persistence.transaction.UnitOfWork
-import org.apache.kylin.engine.spark.job.NSparkCubingUtil._
+import org.apache.kylin.engine.spark.job.NSparkCubingUtil
import org.apache.kylin.metadata.cube.model.{NDataSegment, NDataflowManager, NDataflowUpdate}
-import org.apache.kylin.common.KylinConfig
import org.apache.kylin.metadata.model.TblColRef
import org.apache.spark.internal.Logging
import org.apache.spark.sql.functions.expr
@@ -51,7 +51,7 @@ object DFBuilderHelper extends Logging {
def selectColumnsInTable(table: Dataset[Row], columns: Set[TblColRef]): Set[TblColRef] = {
columns.filter(col =>
- isColumnInTable(convertFromDot(col.getBackTickExpressionInSourceDB), table))
+ isColumnInTable(NSparkCubingUtil.convertFromDot(col.getBackTickExp), table))
}
// ============================= Used by {@link DFBuildJob}.Functions are deprecated. ========================= //
@@ -63,7 +63,7 @@ object DFBuilderHelper extends Logging {
@deprecated
def filterOutIntegerFamilyType(table: Dataset[Row], columns: Set[TblColRef]): Set[TblColRef] = {
columns.filterNot(_.getType.isIntegerFamily).filter(cc =>
- isColumnInTable(convertFromDot(cc.getBackTickExpressionInSourceDB), table))
+ isColumnInTable(NSparkCubingUtil.convertFromDot(cc.getBackTickExp), table))
}
def isColumnInTable(colExpr: String, table: Dataset[Row]): Boolean = {
@@ -78,8 +78,8 @@ object DFBuilderHelper extends Logging {
def chooseSuitableCols(ds: Dataset[Row], needCheckCols: Iterable[TblColRef]): Seq[Column] = {
needCheckCols
.filter(ref => isColumnInTable(ref.getExpressionInSourceDB, ds))
- .map(ref => expr(convertFromDotWithBackTick(ref.getBackTickExpressionInSourceDB))
- .alias(convertFromDot(ref.getBackTickIdentity)))
+ .map(ref => expr(NSparkCubingUtil.convertFromDotWithBackTick(ref.getBackTickExp))
+ .alias(NSparkCubingUtil.convertFromDot(ref.getBackTickIdentity)))
.toSeq
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
index 931570c6c9..7449aebdcd 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
@@ -18,8 +18,12 @@
package org.apache.kylin.engine.spark.builder
-import com.google.common.collect.Sets
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+import java.util.{Locale, Objects, Timer, TimerTask}
+
import org.apache.commons.lang3.StringUtils
+import org.apache.kylin.common.constant.LogConstant
+import org.apache.kylin.common.logging.SetLogCategory
import org.apache.kylin.common.util.HadoopUtil
import org.apache.kylin.common.{CustomUtils, KapConfig, KylinConfig}
import org.apache.kylin.engine.spark.builder.DFBuilderHelper._
@@ -30,18 +34,12 @@ import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.kylin.engine.spark.utils.SparkDataSource._
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.model._
-import org.apache.kylin.query.util.QueryUtil
+import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.utils.ProxyThreadUtils
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-import java.util.{Locale, Objects, Timer, TimerTask}
-
-import org.apache.kylin.common.constant.LogConstant
-import org.apache.kylin.common.logging.SetLogCategory
-import org.apache.spark.util.Utils
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -50,6 +48,8 @@ import scala.concurrent.duration.{Duration, MILLISECONDS}
import scala.concurrent.forkjoin.ForkJoinPool
import scala.util.{Failure, Success, Try}
+import com.google.common.collect.Sets
+
class SegmentFlatTable(private val sparkSession: SparkSession, //
private val tableDesc: SegmentFlatTableDesc) extends LogEx {
@@ -128,7 +128,7 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
val jointDS = joinFactTableWithLookupTables(fastFactTableDS, lookupTableDSMap, dataModel, sparkSession)
concatCCs(jointDS, factTableCCs)
} else {
- fastFactTableDS
+ concatCCs(fastFactTableDS, factTableCCs)
}
flatTableDS = applyFilterCondition(flatTableDS)
changeSchemeToColumnId(flatTableDS, tableDesc)
@@ -258,8 +258,7 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
logInfo(s"No available FILTER-CONDITION segment $segmentId")
return originDS
}
- val expression = QueryUtil.massageExpression(dataModel, project, //
- dataModel.getFilterCondition, null)
+ val expression = PushDownUtil.massageExpression(dataModel, project, dataModel.getFilterCondition, null)
val converted = replaceDot(expression, dataModel)
val condition = s" (1=1) AND ($converted)"
logInfo(s"Apply FILTER-CONDITION: $condition segment $segmentId")
@@ -481,7 +480,7 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
var tableWithCcs = table
matchedCols.foreach(m =>
tableWithCcs = tableWithCcs.withColumn(convertFromDot(m.getBackTickIdentity),
- expr(convertFromDot(m.getBackTickExpressionInSourceDB))))
+ expr(convertFromDot(m.getBackTickExp))))
tableWithCcs
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
index d9af449653..5964c661a6 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
@@ -21,16 +21,15 @@ package org.apache.kylin.engine.spark.job
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.engine.spark.builder.CreateFlatTable.replaceDot
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc
-import org.apache.kylin.query.util.QueryUtil
+import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.internal.Logging
import org.apache.spark.sql.{Dataset, Row}
object FlatTableHelper extends Logging {
- def applyPartitionDesc(
- flatTable: IJoinedFlatTableDesc,
- ds: Dataset[Row],
- needReplaceDot: Boolean): Dataset[Row] = {
+ def applyPartitionDesc(flatTable: IJoinedFlatTableDesc,
+ ds: Dataset[Row],
+ needReplaceDot: Boolean): Dataset[Row] = {
var afterFilter = ds
val model = flatTable.getDataModel
@@ -49,16 +48,15 @@ object FlatTableHelper extends Logging {
afterFilter
}
- def applyFilterCondition(
- flatTable: IJoinedFlatTableDesc,
- ds: Dataset[Row],
- needReplaceDot: Boolean): Dataset[Row] = {
+ def applyFilterCondition(flatTable: IJoinedFlatTableDesc,
+ ds: Dataset[Row],
+ needReplaceDot: Boolean): Dataset[Row] = {
var afterFilter = ds
val model = flatTable.getDataModel
if (StringUtils.isNotBlank(model.getFilterCondition)) {
var filterCond = model.getFilterCondition
- filterCond = QueryUtil.massageExpression(model, model.getProject, filterCond, null);
+ filterCond = PushDownUtil.massageExpression(model, model.getProject, filterCond, null);
if (needReplaceDot) filterCond = replaceDot(filterCond, model)
filterCond = s" (1=1) AND (" + filterCond + s")"
logInfo(s"Filter condition is $filterCond")
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 24391d28a7..1409ff8228 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -18,7 +18,9 @@
package org.apache.kylin.engine.spark.job.stage.build
-import com.google.common.collect.Sets
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+import java.util.{Locale, Objects, Timer, TimerTask}
+
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.Path
import org.apache.kylin.common.util.HadoopUtil
@@ -37,7 +39,7 @@ import org.apache.kylin.engine.spark.utils.SparkDataSource._
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.cube.planner.CostBasePlannerUtils
import org.apache.kylin.metadata.model._
-import org.apache.kylin.query.util.QueryUtil
+import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.sql.KapFunctions.dict_encode_v3
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, expr}
@@ -59,6 +61,8 @@ import scala.concurrent.duration.{Duration, MILLISECONDS}
import scala.concurrent.forkjoin.ForkJoinPool
import scala.util.{Failure, Success, Try}
+import com.google.common.collect.Sets
+
abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
private val dataSegment: NDataSegment,
private val buildParam: BuildParam)
@@ -144,7 +148,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
val jointDS = joinFactTableWithLookupTables(fastFactTableDS, lookupTableDSMap, dataModel, sparkSession)
concatCCs(jointDS, factTableCCs)
} else {
- fastFactTableDS
+ concatCCs(fastFactTableDS, factTableCCs)
}
flatTableDS = applyFilterCondition(flatTableDS)
changeSchemeToColumnId(flatTableDS, tableDesc)
@@ -317,8 +321,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
logInfo(s"No available FILTER-CONDITION segment $segmentId")
return originDS
}
- val expression = QueryUtil.massageExpression(dataModel, project, //
- dataModel.getFilterCondition, null)
+ val expression = PushDownUtil.massageExpression(dataModel, project, dataModel.getFilterCondition, null)
val converted = replaceDot(expression, dataModel)
val condition = s" (1=1) AND ($converted)"
logInfo(s"Apply FILTER-CONDITION: $condition segment $segmentId")
@@ -605,7 +608,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
var tableWithCcs = table
matchedCols.foreach(m =>
tableWithCcs = tableWithCcs.withColumn(convertFromDot(m.getBackTickIdentity),
- expr(convertFromDot(m.getBackTickExpressionInSourceDB))))
+ expr(convertFromDot(m.getBackTickExp))))
tableWithCcs
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
index b4f57ac47f..e1335978fc 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
@@ -20,14 +20,13 @@ package org.apache.kylin.engine.spark.smarter
import java.util
import java.util.Collections
-import com.google.common.collect.{Lists, Maps, Sets}
+import org.apache.commons.collections.CollectionUtils
+import org.apache.commons.lang3.StringUtils
import org.apache.kylin.engine.spark.builder.SegmentFlatTable
import org.apache.kylin.engine.spark.job.NSparkCubingUtil
import org.apache.kylin.metadata.cube.model.LayoutEntity
-import org.apache.kylin.metadata.model.NDataModel
-import org.apache.commons.collections.CollectionUtils
-import org.apache.commons.lang3.StringUtils
-import org.apache.kylin.metadata.model.{FunctionDesc, JoinTableDesc, TableRef, TblColRef}
+import org.apache.kylin.metadata.model.{FunctionDesc, JoinTableDesc, NDataModel, TableRef, TblColRef}
+import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.{Dataset, Row, SparderEnv, SparkSession}
@@ -35,6 +34,8 @@ import org.apache.spark.sql.{Dataset, Row, SparderEnv, SparkSession}
import scala.collection.JavaConverters._
import scala.collection.mutable
+import com.google.common.collect.{Lists, Maps, Sets}
+
class IndexDependencyParser(val model: NDataModel) {
private val ccTableNameAliasMap = Maps.newHashMap[String, util.Set[String]]
@@ -105,7 +106,7 @@ class IndexDependencyParser(val model: NDataModel) {
}
}
- private def generateFullFlatTableDF(ss: SparkSession, model: NDataModel): Dataset[Row] = {
+ def generateFullFlatTableDF(ss: SparkSession, model: NDataModel): Dataset[Row] = {
val rootDF = generateDatasetOnTable(ss, model.getRootFactTable)
// look up tables
val joinTableDFMap = mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]]()
@@ -113,14 +114,16 @@ class IndexDependencyParser(val model: NDataModel) {
joinTableDFMap.put(joinTable, generateDatasetOnTable(ss, joinTable.getTableRef))
})
val df = SegmentFlatTable.joinFactTableWithLookupTables(rootDF, joinTableDFMap, model, ss)
- if (StringUtils.isNotEmpty(model.getFilterCondition)) {
- df.where(NSparkCubingUtil.convertFromDotWithBackTick(model.getFilterCondition))
+ val filterCondition = model.getFilterCondition
+ if (StringUtils.isNotEmpty(filterCondition)) {
+ val massagedCondition = PushDownUtil.massageExpression(model, model.getProject, filterCondition, null)
+ df.where(NSparkCubingUtil.convertFromDotWithBackTick(massagedCondition))
}
df
}
private def generateDatasetOnTable(ss: SparkSession, tableRef: TableRef): Dataset[Row] = {
- val tableCols = tableRef.getColumns.asScala.map(_.getColumnDesc).filter(!_.isComputedColumn).toArray
+ val tableCols = tableRef.getColumns.asScala.map(_.getColumnDesc).toArray
val structType = SchemaProcessor.buildSchemaWithRawTable(tableCols)
val alias = tableRef.getAlias
val dataset = ss.createDataFrame(Lists.newArrayList[Row], structType).alias(alias)
@@ -190,7 +193,7 @@ class IndexDependencyParser(val model: NDataModel) {
}
}
- private def initJoinTableName() {
+ private def initJoinTableName(): Unit = {
if (CollectionUtils.isEmpty(model.getJoinTables)) {
return
}
diff --git a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java
index 4b53b720c0..de3a0d8f4d 100644
--- a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java
+++ b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java
@@ -52,7 +52,7 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner {
@Override
public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas,
- String project) throws Exception {
+ String project) throws SQLException {
Statement statement = null;
Connection connection = manager.getConnection();
ResultSet resultSet = null;
@@ -83,7 +83,7 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner {
}
@Override
- public void executeUpdate(String sql, String project) throws Exception {
+ public void executeUpdate(String sql, String project) throws SQLException {
Statement statement = null;
Connection connection = manager.getConnection();
diff --git a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java
index 81887d9c8f..4cf2543e7e 100644
--- a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java
+++ b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.pushdown;
+import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
@@ -45,7 +46,7 @@ public class PushDownRunnerSparkImpl implements IPushDownRunner {
@Override
public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas,
- String project) {
+ String project) throws SQLException {
PushdownResult response = executeQueryToIterator(query, project);
response.getRows().forEach(results::add);
columnMetas.addAll(response.getColumnMetas());
diff --git a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java
index 5694f11483..f8a24e3077 100644
--- a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java
+++ b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.pushdown;
+import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -99,14 +100,18 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
queries.forEach(q -> {
returnRows.clear();
- pushDownRunnerSpark.executeQuery(q, returnRows, returnColumnMeta, "tpch");
+ try {
+ pushDownRunnerSpark.executeQuery(q, returnRows, returnColumnMeta, "tpch");
+ } catch (SQLException e) {
+ //
+ }
Assert.assertEquals(10, returnRows.size());
});
}
@Test
- public void testPushDownRunnerSpark() {
+ public void testPushDownRunnerSpark() throws SQLException {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
pushDownRunnerSpark.init(null, "tpch");
@@ -122,7 +127,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testPushDownRunnerSparkWithDotColumn() {
+ public void testPushDownRunnerSparkWithDotColumn() throws SQLException {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
pushDownRunnerSpark.init(null, "tpch");
@@ -138,7 +143,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testSelectTwoSameExpr() {
+ public void testSelectTwoSameExpr() throws SQLException {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
pushDownRunnerSpark.init(null, "tpch");
@@ -154,7 +159,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testCaseSensitiveOnAlias() {
+ public void testCaseSensitiveOnAlias() throws SQLException {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
pushDownRunnerSpark.init(null, "tpch");
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
index c22178e9d7..89a0f316a3 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
@@ -20,7 +20,6 @@ package org.apache.kylin.engine.spark.job;
import java.util.Arrays;
import java.util.LinkedHashSet;
-import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Matcher;
@@ -38,10 +37,6 @@ import org.apache.kylin.metadata.model.Segments;
import org.apache.spark.sql.Column;
import org.sparkproject.guava.collect.Sets;
-import com.google.common.collect.Maps;
-
-import lombok.val;
-
public class NSparkCubingUtil {
public static final String SEPARATOR = "_0_DOT_0_";
@@ -285,10 +280,4 @@ public class NSparkCubingUtil {
return withoutDot.replace(SEPARATOR, ".");
}
- public static Map<Long, LayoutEntity> toLayoutMap(IndexPlan indexPlan, Set<Long> layoutIds) {
- val layouts = toLayouts(indexPlan, layoutIds).stream().filter(Objects::nonNull).collect(Collectors.toSet());
- Map<Long, LayoutEntity> map = Maps.newHashMap();
- layouts.forEach(layout -> map.put(layout.getId(), layout));
- return map;
- }
}
diff --git a/src/spark-project/spark-it/src/test/scala/io/kyligence/kap/common/SSSource.scala b/src/spark-project/spark-it/src/test/scala/io/kyligence/kap/common/SSSource.scala
new file mode 100644
index 0000000000..f071e36acc
--- /dev/null
+++ b/src/spark-project/spark-it/src/test/scala/io/kyligence/kap/common/SSSource.scala
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * http://kyligence.io
+ * This software is the confidential and proprietary information of
+ * Kyligence Inc. ("Confidential Information"). You shall not disclose
+ * such Confidential Information and shall use it only in accordance
+ * with the terms of the license agreement you entered into with
+ * Kyligence Inc.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+package io.kyligence.kap.common
+
+import java.util.Locale
+
+import org.apache.kylin.common.KylinConfig
+import org.apache.kylin.common.util.TempMetadataBuilder
+import org.apache.kylin.metadata.model.NTableMetadataManager
+import org.apache.kylin.metadata.project.NProjectManager
+import org.apache.kylin.query.util.{PushDownUtil, QueryParams}
+import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession}
+import org.apache.spark.sql.execution.utils.SchemaProcessor
+import org.scalatest.Suite
+
+import com.google.common.base.Preconditions
+
+trait SSSource extends SharedSparkSession with LocalMetadata {
+ self: Suite =>
+
+ val CSV_TABLE_DIR: String = "../" + TempMetadataBuilder.TEMP_TEST_METADATA + "/data/%s.csv"
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ val project = getProject
+ import org.apache.kylin.metadata.project.NProjectManager
+ val kylinConf = KylinConfig.getInstanceFromEnv
+ val projectInstance =
+ NProjectManager.getInstance(kylinConf).getProject(project)
+ Preconditions.checkArgument(projectInstance != null)
+ import scala.collection.JavaConverters._
+ projectInstance.getTables.asScala
+ .filter(!_.equals("DEFAULT.STREAMING_TABLE"))
+ .foreach { table =>
+ val tableDesc = NTableMetadataManager
+ .getInstance(kylinConf, project)
+ .getTableDesc(table)
+ val columns = tableDesc.getColumns
+ val schema = SchemaProcessor.buildSchemaWithRawTable(columns)
+ var tableN = tableDesc.getName
+ if (table.equals("DEFAULT.TEST_KYLIN_FACT")) {
+ tableN = tableDesc.getName + "_table"
+ }
+ spark.catalog.createTable(
+ tableName = tableN,
+ source = "csv",
+ schema = schema,
+ options = Map("path" -> String.format(Locale.ROOT, CSV_TABLE_DIR, table)))
+ if (table.equals("DEFAULT.TEST_KYLIN_FACT")) {
+ spark.sql("create view " + tableDesc.getName + " as select * from " + tableN)
+ }
+ }
+ }
+
+ protected def getProject: String = "default"
+
+ def cleanSql(originSql: String): String = {
+ val sqlForSpark = originSql
+ .replaceAll("edw\\.", "")
+ .replaceAll("\"EDW\"\\.", "")
+ .replaceAll("EDW\\.", "")
+ .replaceAll("default\\.", "")
+ .replaceAll("DEFAULT\\.", "")
+ .replaceAll("\"DEFAULT\"\\.", "")
+ val queryParams = new QueryParams("default", sqlForSpark, "DEFAULT", false)
+ queryParams.setKylinConfig(NProjectManager.getProjectConfig("default"))
+ PushDownUtil.massagePushDownSql(queryParams)
+ }
+}
diff --git a/src/streaming/src/main/scala/org/apache/kylin/streaming/CreateStreamingFlatTable.scala b/src/streaming/src/main/scala/org/apache/kylin/streaming/CreateStreamingFlatTable.scala
index 13fb54a288..909f39957e 100644
--- a/src/streaming/src/main/scala/org/apache/kylin/streaming/CreateStreamingFlatTable.scala
+++ b/src/streaming/src/main/scala/org/apache/kylin/streaming/CreateStreamingFlatTable.scala
@@ -18,9 +18,9 @@
package org.apache.kylin.streaming
-import com.google.common.base.Preconditions
+import java.nio.ByteBuffer
+
import org.apache.commons.lang3.StringUtils
-import org.apache.kafka.common.config.SaslConfigs
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.engine.spark.NSparkCubingEngine
import org.apache.kylin.engine.spark.builder.CreateFlatTable
@@ -31,17 +31,17 @@ import org.apache.kylin.metadata.model._
import org.apache.kylin.parser.AbstractDataParser
import org.apache.kylin.source.SourceFactory
import org.apache.kylin.streaming.common.CreateFlatTableEntry
-import org.apache.kylin.streaming.jobs.StreamingJobUtils
import org.apache.spark.sql.catalyst.encoders.RowEncoder
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.sql.{DataFrame, Dataset, Row}
import org.apache.spark.storage.StorageLevel
-import java.nio.ByteBuffer
import scala.collection.JavaConverters._
import scala.collection.mutable
+import com.google.common.base.Preconditions
+
class CreateStreamingFlatTable(entry: CreateFlatTableEntry) extends
CreateFlatTable(entry.flatTable, entry.seg, entry.toBuildTree, entry.ss, entry.sourceInfo) {
@@ -63,12 +63,12 @@ class CreateStreamingFlatTable(entry: CreateFlatTableEntry) extends
val kafkaJobParams = config.getStreamingKafkaConfigOverride.asScala
val securityProtocol = kafkaJobParams.get(SECURITY_PROTOCOL)
if (securityProtocol.isDefined) {
- kafkaJobParams.remove(SECURITY_PROTOCOL);
+ kafkaJobParams.remove(SECURITY_PROTOCOL)
kafkaJobParams.put("kafka." + SECURITY_PROTOCOL, securityProtocol.get)
}
val saslMechanism = kafkaJobParams.get(SASL_MECHANISM)
if (saslMechanism.isDefined) {
- kafkaJobParams.remove(SASL_MECHANISM);
+ kafkaJobParams.remove(SASL_MECHANISM)
kafkaJobParams.put("kafka." + SASL_MECHANISM, saslMechanism.get)
}
kafkaJobParams.foreach { param =>
@@ -91,7 +91,7 @@ class CreateStreamingFlatTable(entry: CreateFlatTableEntry) extends
val schema =
StructType(
tableDesc.getColumns.map { columnDescs =>
- StructField(columnDescs.getName, SparderTypeUtil.toSparkType(columnDescs.getType, false))
+ StructField(columnDescs.getName, SparderTypeUtil.toSparkType(columnDescs.getType))
}
)
val rootFactTable = changeSchemaToAliasDotName(
@@ -104,11 +104,11 @@ class CreateStreamingFlatTable(entry: CreateFlatTableEntry) extends
val cols = model.getRootFactTable.getColumns.asScala.map(item => {
col(NSparkCubingUtil.convertFromDot(item.getBackTickIdentity))
}).toList
- rootFactTable.withWatermark(partitionColumn, entry.watermark).groupBy(cols: _*).count()
+ rootFactTable.withWatermark(partitionColumn(), entry.watermark).groupBy(cols: _*).count()
} else {
rootFactTable
}
- tableRefreshInterval = StreamingUtils.parseTableRefreshInterval(config.getStreamingTableRefreshInterval())
+ tableRefreshInterval = StreamingUtils.parseTableRefreshInterval(config.getStreamingTableRefreshInterval)
loadLookupTables()
joinFactTableWithLookupTables(factTableDataset, lookupTablesGlobal, model, ss)
}
@@ -116,7 +116,7 @@ class CreateStreamingFlatTable(entry: CreateFlatTableEntry) extends
def loadLookupTables(): Unit = {
val ccCols = model().getRootFactTable.getColumns.asScala.filter(_.getColumnDesc.isComputedColumn).toSet
val cleanLookupCC = cleanComputColumn(ccCols.toSeq, factTableDataset.columns.toSet)
- lookupTablesGlobal = generateLookupTableDataset(model, cleanLookupCC, ss)
+ lookupTablesGlobal = generateLookupTableDataset(model(), cleanLookupCC, ss)
lookupTablesGlobal.foreach { case (_, df) =>
df.persist(StorageLevel.MEMORY_AND_DISK)
}
@@ -138,10 +138,10 @@ class CreateStreamingFlatTable(entry: CreateFlatTableEntry) extends
}
def encodeStreamingDataset(seg: NDataSegment, model: NDataModel, batchDataset: Dataset[Row]): Dataset[Row] = {
- val ccCols = model.getRootFactTable.getColumns.asScala.filter(_.getColumnDesc.isComputedColumn).toSet
+ val ccCols = model.getRootFactTable.getColumns.asScala.toSet
val (dictCols, encodeCols): GlobalDictType = assemblyGlobalDictTuple(seg, toBuildTree)
val encodedDataset = encodeWithCols(batchDataset, ccCols, dictCols, encodeCols)
- val filterEncodedDataset = FlatTableHelper.applyFilterCondition(flatTable, encodedDataset, true)
+ val filterEncodedDataset = FlatTableHelper.applyFilterCondition(flatTable, encodedDataset, needReplaceDot = true)
flatTable match {
case joined: NCubeJoinedFlatTableDesc =>
[kylin] 13/38: KYLIN-5528 support collect sparder gc
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 842a0601a643ac83d894ec8a248673c8fd13cfb5
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Thu Feb 23 18:00:57 2023 +0800
KYLIN-5528 support collect sparder gc
* KYLIN-5528 support collect sparder gc
* KYLIN-5528 add gc option to executor extraJavaOption
* KYLIN-5528 change timeout to 30s
---
.../src/main/resources/config/init.properties | 2 +-
.../apache/kylin/rest/service/SystemService.java | 5 +
.../org/apache/kylin/common/KylinConfigBase.java | 14 +++
.../src/main/resources/kylin-defaults0.properties | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../kylin/rest/controller/NQueryController.java | 2 +-
.../src/main/resources/config/init.properties | 2 +-
.../engine/spark/job/NSparkExecutableTest.java | 15 ++-
.../asyncprofiler/AsyncProfiling.scala | 2 +-
.../QueryAsyncProfilerDriverPlugin.scala | 2 +-
.../QueryAsyncProfilerSparkPlugin.scala | 2 +-
.../diagnose/DiagnoseConstant.scala} | 21 +++-
.../diagnose/DiagnoseDriverPlugin.scala} | 50 +++++----
.../plugin/diagnose/DiagnoseExecutorPlugin.scala | 124 +++++++++++++++++++++
.../query/plugin/diagnose/DiagnoseHelper.scala | 62 +++++++++++
.../diagnose/DiagnoseSparkPlugin.scala} | 9 +-
.../scala/org/apache/spark/sql/KylinSession.scala | 56 ++++++----
.../scala/org/apache/spark/sql/SparderEnv.scala | 5 +-
.../SparkPluginWithMeta.scala} | 4 +-
.../asyncprofiler/AsyncProfilingTest.scala | 5 +-
.../QueryAsyncProfilerSparkPluginTest.scala | 2 +-
.../QueryProfilerDriverPluginTest.scala} | 5 +-
.../diagnose/DiagnoseExecutorPluginTest.scala | 109 ++++++++++++++++++
.../query/plugin/diagnose/DiagnosePluginTest.scala | 82 ++++++++++++++
25 files changed, 506 insertions(+), 80 deletions(-)
diff --git a/src/common-booter/src/main/resources/config/init.properties b/src/common-booter/src/main/resources/config/init.properties
index 5dbf475978..3ff1076256 100644
--- a/src/common-booter/src/main/resources/config/init.properties
+++ b/src/common-booter/src/main/resources/config/init.properties
@@ -113,7 +113,7 @@ kylin.storage.columnar.spark-env.HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
# for any spark config entry in http://spark.apache.org/docs/latest/configuration.html#spark-properties, prefix it with "kap.storage.columnar.spark-conf" and append here
-kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M
+kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTime [...]
kylin.storage.columnar.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-appmaster-log4j.xml
kylin.storage.columnar.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.storage.columnar.spark-conf.spark.serializer=org.apache.spark.serializer.JavaSerializer
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
index fa467e414d..25c961be8c 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
@@ -60,6 +60,7 @@ import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.plugin.diagnose.DiagnoseHelper;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.BackupRequest;
import org.apache.kylin.rest.request.DiagProgressRequest;
@@ -138,6 +139,8 @@ public class SystemService extends BasicService {
String[] arguments;
// full
if (StringUtils.isEmpty(jobId) && StringUtils.isEmpty(queryId)) {
+ // Sparder executor gc log should be collected before FULL and QUERY diag package
+ DiagnoseHelper.collectSparderExecutorGc();
if (startTime == null && endTime == null) {
startTime = Long.toString(System.currentTimeMillis() - 259200000L);
endTime = Long.toString(System.currentTimeMillis());
@@ -153,6 +156,8 @@ public class SystemService extends BasicService {
arguments = new String[] { jobOpt, jobId, "-destDir", exportFile.getAbsolutePath(), "-diagId", uuid };
diagPackageType = JOB;
} else { //query
+ // Sparder executor gc log should be collected before FULL and QUERY diag package
+ DiagnoseHelper.collectSparderExecutorGc();
arguments = new String[] { "-project", project, "-query", queryId, "-destDir", exportFile.getAbsolutePath(),
"-diagId", uuid };
diagPackageType = QUERY;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 32af04760a..0490ce7b84 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2565,6 +2565,10 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.job.notification-on-empty-data-load", FALSE));
}
+ public boolean getJobErrorNotificationEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.job.notification-on-job-error", FALSE));
+ }
+
public Long getStorageResourceSurvivalTimeThreshold() {
return TimeUtil.timeStringAs(this.getOptional("kylin.storage.resource-survival-time-threshold", "7d"),
TimeUnit.MILLISECONDS);
@@ -3812,6 +3816,16 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.build.allow-non-strict-count-check", FALSE));
}
+ public long queryDiagnoseCollectionTimeout() {
+ return TimeUtil.timeStringAs(getOptional("kylin.query.diagnose-collection-timeout", "30s"),
+ TimeUnit.MILLISECONDS);
+ }
+
+ public boolean queryDiagnoseEnable() {
+ return !Boolean.parseBoolean(System.getProperty("spark.local", FALSE))
+ && Boolean.parseBoolean(getOptional("kylin.query.diagnose-enabled", TRUE));
+ }
+
// ============================================================================
// Cost based index Planner
// ============================================================================
diff --git a/src/core-common/src/main/resources/kylin-defaults0.properties b/src/core-common/src/main/resources/kylin-defaults0.properties
index cbacd5d78a..9aee2acdda 100644
--- a/src/core-common/src/main/resources/kylin-defaults0.properties
+++ b/src/core-common/src/main/resources/kylin-defaults0.properties
@@ -115,7 +115,7 @@ kylin.storage.columnar.spark-env.HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
# for any spark config entry in http://spark.apache.org/docs/latest/configuration.html#spark-properties, prefix it with "kap.storage.columnar.spark-conf" and append here
-kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M
+kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTime [...]
kylin.storage.columnar.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-appmaster-log4j.xml
kylin.storage.columnar.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.storage.columnar.spark-conf.spark.serializer=org.apache.spark.serializer.JavaSerializer
diff --git a/src/data-loading-booter/src/main/resources/config/init.properties b/src/data-loading-booter/src/main/resources/config/init.properties
index 5dbf475978..3ff1076256 100644
--- a/src/data-loading-booter/src/main/resources/config/init.properties
+++ b/src/data-loading-booter/src/main/resources/config/init.properties
@@ -113,7 +113,7 @@ kylin.storage.columnar.spark-env.HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
# for any spark config entry in http://spark.apache.org/docs/latest/configuration.html#spark-properties, prefix it with "kap.storage.columnar.spark-conf" and append here
-kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M
+kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTime [...]
kylin.storage.columnar.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-appmaster-log4j.xml
kylin.storage.columnar.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.storage.columnar.spark-conf.spark.serializer=org.apache.spark.serializer.JavaSerializer
diff --git a/src/query-booter/src/main/resources/config/init.properties b/src/query-booter/src/main/resources/config/init.properties
index 5dbf475978..3ff1076256 100644
--- a/src/query-booter/src/main/resources/config/init.properties
+++ b/src/query-booter/src/main/resources/config/init.properties
@@ -113,7 +113,7 @@ kylin.storage.columnar.spark-env.HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
# for any spark config entry in http://spark.apache.org/docs/latest/configuration.html#spark-properties, prefix it with "kap.storage.columnar.spark-conf" and append here
-kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M
+kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTime [...]
kylin.storage.columnar.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-appmaster-log4j.xml
kylin.storage.columnar.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.storage.columnar.spark-conf.spark.serializer=org.apache.spark.serializer.JavaSerializer
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index ea088d38a8..21936b8faf 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -71,7 +71,7 @@ import org.apache.kylin.common.persistence.transaction.StopQueryBroadcastEventNo
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.metadata.query.QueryHistoryRequest;
import org.apache.kylin.metadata.query.util.QueryHisTransformStandardUtil;
-import org.apache.kylin.query.asyncprofiler.AsyncProfiling;
+import org.apache.kylin.query.plugin.asyncprofiler.AsyncProfiling;
import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.request.SQLFormatRequest;
import org.apache.kylin.rest.response.QueryStatisticsResponse;
diff --git a/src/server/src/main/resources/config/init.properties b/src/server/src/main/resources/config/init.properties
index 5dbf475978..3ff1076256 100644
--- a/src/server/src/main/resources/config/init.properties
+++ b/src/server/src/main/resources/config/init.properties
@@ -113,7 +113,7 @@ kylin.storage.columnar.spark-env.HADOOP_CONF_DIR=${kylin_hadoop_conf_dir}
# for any spark config entry in http://spark.apache.org/docs/latest/configuration.html#spark-properties, prefix it with "kap.storage.columnar.spark-conf" and append here
-kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M
+kylin.storage.columnar.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-executor-log4j.xml -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkap.metadata.identifier=${kylin.metadata.url.identifier} -Dkap.spark.category=sparder -Dkap.spark.project=${job.project} -Dkap.spark.mountDir=${kylin.tool.mount-spark-log-dir} -XX:MaxDirectMemorySize=896M -XX:+PrintFlagsFinal -XX:+PrintReferenceGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTime [...]
kylin.storage.columnar.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current -Dlog4j.configurationFile=spark-appmaster-log4j.xml
kylin.storage.columnar.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.storage.columnar.spark-conf.spark.serializer=org.apache.spark.serializer.JavaSerializer
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkExecutableTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkExecutableTest.java
index ce5a26c72d..2fcf39b87d 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkExecutableTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkExecutableTest.java
@@ -29,6 +29,8 @@ import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin;
+import org.apache.kylin.query.plugin.asyncprofiler.QueryAsyncProfilerSparkPlugin;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -152,12 +154,11 @@ public class NSparkExecutableTest extends NLocalFileMetadataTestCase {
String cmd = (String) sparkExecutable.sparkJobHandler.generateSparkCmd(kylinConfig, desc);
Assert.assertNotNull(cmd);
- Assert.assertTrue(
- cmd.contains("spark.plugins=org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin"));
+ Assert.assertTrue(cmd.contains("spark.plugins=" + BuildAsyncProfilerSparkPlugin.class.getCanonicalName()));
}
overwriteSystemProp("kylin.engine.spark-conf.spark.plugins",
- "org.apache.kylin.query.asyncprofiler.QueryAsyncProfilerSparkPlugin");
+ QueryAsyncProfilerSparkPlugin.class.getCanonicalName());
{
val desc = sparkExecutable.getSparkAppDesc();
desc.setHadoopConfDir(hadoopConf);
@@ -166,9 +167,8 @@ public class NSparkExecutableTest extends NLocalFileMetadataTestCase {
String cmd = (String) sparkExecutable.sparkJobHandler.generateSparkCmd(kylinConfig, desc);
Assert.assertNotNull(cmd);
- Assert.assertTrue(
- cmd.contains("spark.plugins=org.apache.kylin.query.asyncprofiler.QueryAsyncProfilerSparkPlugin,"
- + "org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin"));
+ Assert.assertTrue(cmd.contains("spark.plugins=" + QueryAsyncProfilerSparkPlugin.class.getCanonicalName()
+ + "," + BuildAsyncProfilerSparkPlugin.class.getCanonicalName()));
}
overwriteSystemProp("kylin.engine.async-profiler-enabled", "false");
@@ -180,8 +180,7 @@ public class NSparkExecutableTest extends NLocalFileMetadataTestCase {
String cmd = (String) sparkExecutable.sparkJobHandler.generateSparkCmd(kylinConfig, desc);
Assert.assertNotNull(cmd);
- Assert.assertFalse(
- cmd.contains("spark.plugins=org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin"));
+ Assert.assertFalse(cmd.contains("spark.plugins=" + BuildAsyncProfilerSparkPlugin.class.getCanonicalName()));
}
overwriteSystemProp("kylin.engine.spark-conf.spark.driver.extraJavaOptions",
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/AsyncProfiling.scala
similarity index 98%
rename from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala
rename to src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/AsyncProfiling.scala
index d27cfc76f5..8147ba94ec 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/AsyncProfiling.scala
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.asyncprofiler
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.asyncprofiler.Message._
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
similarity index 97%
copy from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
copy to src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
index 8af9631561..f8fe0e38a6 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.asyncprofiler
import org.apache.kylin.common.asyncprofiler.AsyncProfilerTool
import org.apache.spark.SparkContext
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
similarity index 95%
copy from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
copy to src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
index fd4306d3af..a2e4af8554 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.asyncprofiler
import org.apache.kylin.common.asyncprofiler.AsyncProfilerExecutorPlugin
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, SparkPlugin}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseConstant.scala
similarity index 66%
copy from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
copy to src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseConstant.scala
index fd4306d3af..98c3cbf585 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseConstant.scala
@@ -16,14 +16,23 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.diagnose
-import org.apache.kylin.common.asyncprofiler.AsyncProfilerExecutorPlugin
-import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, SparkPlugin}
+object DiagnoseConstant {
+ // state
+ val STATE_WAIT = "STATE_WAIT"
+ val STATE_COLLECT = "STATE_COLLECT"
-class QueryAsyncProfilerSparkPlugin extends SparkPlugin {
+ // executor message
+ val NEXTCMD = "NEXTCMD"
+ val SENDRESULT = "SENDRESULT"
+ val HDFSDIR = "HDFSDIR"
- override def driverPlugin(): DriverPlugin = new QueryAsyncProfilerDriverPlugin
+ // driver message
+ val NOP = "NOP"
+ val COLLECT = "COLLECT"
+
+ // empty
+ val EMPTY = ""
- override def executorPlugin(): ExecutorPlugin = new AsyncProfilerExecutorPlugin
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseDriverPlugin.scala
similarity index 51%
rename from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
rename to src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseDriverPlugin.scala
index 8af9631561..fed11cd45e 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseDriverPlugin.scala
@@ -16,34 +16,36 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.diagnose
-import org.apache.kylin.common.asyncprofiler.AsyncProfilerTool
-import org.apache.spark.SparkContext
-import org.apache.spark.api.plugin.{DriverPlugin, PluginContext}
+import org.apache.kylin.common.KylinConfig
+import org.apache.spark.api.plugin.DriverPlugin
import org.apache.spark.internal.Logging
-import java.util
-
-class QueryAsyncProfilerDriverPlugin extends DriverPlugin with Logging {
-
- override def init(sc: SparkContext, pluginContext: PluginContext): util.Map[String, String] = {
- // Sparder Driver and KE are always in one JVM, in client mode
- AsyncProfilerTool.loadAsyncProfilerLib(true)
- super.init(sc, pluginContext)
- }
-
+class DiagnoseDriverPlugin extends DriverPlugin with Logging {
override def receive(message: Any): AnyRef = {
- import org.apache.kylin.common.asyncprofiler.Message._
+ message match {
+ case DiagnoseConstant.NEXTCMD =>
+ getNextCommand()
+ case DiagnoseConstant.HDFSDIR =>
+ KylinConfig.getInstanceFromEnv.getHdfsWorkingDirectory
+ case DiagnoseConstant.SENDRESULT =>
+ countDownGcResult()
+ case _ => DiagnoseConstant.EMPTY
+ }
+ }
- val (command, executorId, param) = processMessage(message.toString)
- command match {
- case NEXT_COMMAND =>
- AsyncProfiling.nextCommand()
- case RESULT =>
- AsyncProfiling.cacheExecutorResult(param, executorId)
- ""
- case _ => ""
+ def getNextCommand(): AnyRef = {
+ if (DiagnoseConstant.STATE_COLLECT.equals(DiagnoseHelper.state)) {
+ DiagnoseConstant.COLLECT
+ } else {
+ DiagnoseConstant.NOP
}
}
-}
\ No newline at end of file
+
+ def countDownGcResult(): AnyRef = {
+ DiagnoseHelper.countDownGcResult()
+ DiagnoseConstant.EMPTY
+ }
+
+}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPlugin.scala
new file mode 100644
index 0000000000..c6c811b7c3
--- /dev/null
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPlugin.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.plugin.diagnose
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.kylin.common.util.ExecutorServiceUtil
+import org.apache.spark.api.plugin.{ExecutorPlugin, PluginContext}
+import org.apache.spark.internal.Logging
+import org.joda.time.DateTime
+
+import java.io.File
+import java.util
+import java.util.concurrent.{Executors, TimeUnit}
+
+class DiagnoseExecutorPlugin extends ExecutorPlugin with Logging {
+
+ private val SPARDER_LOG: String = "_sparder_logs"
+ private val LOCAL_GC_FILE_PREFIX: String = "gc"
+ private val DATE_PATTERN = "yyyy-MM-dd"
+ private val checkingInterval: Long = 10000L
+ private val configuration: Configuration = new Configuration()
+ private val fileSystem: FileSystem = FileSystem.get(configuration)
+
+ private val scheduledExecutorService = Executors.newScheduledThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Diagnose-%d").build())
+
+ private var state = DiagnoseConstant.STATE_WAIT
+ private var curContainerDir = new File(".")
+ private var sparderLogDir: String = ""
+ private var ctx: PluginContext = _
+
+ override def init(_ctx: PluginContext, extraConf: util.Map[String, String]): Unit = {
+ ctx = _ctx
+ val diagnose = new Runnable {
+ override def run(): Unit = checkAndDiagnose()
+ }
+ logInfo("Diagnose executor plugin is initializing ...")
+ scheduledExecutorService.scheduleWithFixedDelay(
+ diagnose, 0, checkingInterval, TimeUnit.MILLISECONDS)
+ }
+
+ def checkAndDiagnose(): Unit = {
+ try {
+ val replay: AnyRef = ctx.ask(DiagnoseConstant.NEXTCMD)
+ logDebug(s"Executor ${ctx.executorID()} get replay $replay from driver ...")
+ replay match {
+ case DiagnoseConstant.COLLECT =>
+ if (DiagnoseConstant.STATE_WAIT.equals(state)) {
+ state = DiagnoseConstant.STATE_COLLECT
+ logDebug(s"Set executor state to $state")
+ collectGcLog()
+ ctx.send(DiagnoseConstant.SENDRESULT)
+ }
+ case DiagnoseConstant.NOP =>
+ if (!DiagnoseConstant.STATE_WAIT.equals(state)) {
+ state = DiagnoseConstant.STATE_WAIT
+ logDebug(s"Set executor state to $state")
+ }
+ case _ => ""
+ }
+ } catch {
+ case e: Exception =>
+ logInfo("Error while communication/Diagnose", e)
+ }
+ }
+
+ def collectGcLog(): Unit = {
+ logDebug(s"Collectting sparder gc log file ...")
+ if (sparderLogDir.isEmpty) {
+ val reply = ctx.ask(DiagnoseConstant.HDFSDIR).toString
+ if (reply.isEmpty) {
+ logWarning(s"Can not get kylin working dir, will not collect sparder executor gc log.")
+ return
+ } else {
+ sparderLogDir = reply + SPARDER_LOG
+ logInfo(s"HDFS sparder log dir is setting to ${sparderLogDir}")
+ }
+ }
+ val filePath = sparderLogDir + File.separator + new DateTime().toString(DATE_PATTERN) +
+ File.separator + ctx.conf().getAppId + File.separator
+ val fileNamePrefix = "executor-%s-".format(ctx.executorID())
+
+ curContainerDir.listFiles().filter(file => file.getName.startsWith(LOCAL_GC_FILE_PREFIX))
+ .map(file => copyLocalFileToHdfs(new Path(file.getAbsolutePath), new Path(filePath, fileNamePrefix + file.getName)))
+ }
+
+ def copyLocalFileToHdfs(local: Path, hdfs: Path): Unit = {
+ logInfo(s"Local gc file path is: ${local}, target hdfs file is: ${hdfs}")
+ fileSystem.copyFromLocalFile(local, hdfs)
+ }
+
+ override def shutdown(): Unit = {
+ ExecutorServiceUtil.shutdownGracefully(scheduledExecutorService, 3)
+ super.shutdown()
+ }
+
+ // for test only
+ def setCtx(_ctx: PluginContext): Unit = {
+ ctx = _ctx
+ }
+
+ // for test only
+ def setContainerDir(_curContainerDir: File): Unit = {
+ curContainerDir = _curContainerDir
+ }
+}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseHelper.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseHelper.scala
new file mode 100644
index 0000000000..9e0c3d2133
--- /dev/null
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseHelper.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.plugin.diagnose
+
+import org.apache.kylin.common.KylinConfig
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparderEnv
+
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+object DiagnoseHelper extends Logging {
+ var state: String = DiagnoseConstant.STATE_WAIT
+ var resultCollectionTimeout: Long = KylinConfig.getInstanceFromEnv.queryDiagnoseCollectionTimeout
+ var gcResult: CountDownLatch = _
+
+ // activeExecutorCount will be set for UT
+ var activeExecutorCount: Int = 0
+
+ def collectSparderExecutorGc(): Unit = {
+ if (!KylinConfig.getInstanceFromEnv.isUTEnv) {
+ activeExecutorCount = SparderEnv.getActiveExecutorIds.size
+ }
+ initGcCount(activeExecutorCount)
+
+ setState(DiagnoseConstant.STATE_COLLECT)
+
+ if (gcResult.await(resultCollectionTimeout, TimeUnit.MILLISECONDS)) {
+ logInfo("All executor gc logs have been uploaded to hdfs")
+ } else {
+ logWarning("Timeout while waiting for gc log result")
+ }
+ setState(DiagnoseConstant.STATE_WAIT)
+ }
+
+ def initGcCount(count: Int): Unit = {
+ gcResult = new CountDownLatch(count)
+ }
+
+ def setState(_state: String): Unit = {
+ DiagnoseHelper.state = _state
+ }
+
+ def countDownGcResult(): Unit = {
+ gcResult.countDown()
+ }
+}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseSparkPlugin.scala
similarity index 72%
rename from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
rename to src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseSparkPlugin.scala
index fd4306d3af..52a90e93de 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPlugin.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseSparkPlugin.scala
@@ -16,14 +16,13 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.diagnose
-import org.apache.kylin.common.asyncprofiler.AsyncProfilerExecutorPlugin
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, SparkPlugin}
-class QueryAsyncProfilerSparkPlugin extends SparkPlugin {
+class DiagnoseSparkPlugin extends SparkPlugin {
- override def driverPlugin(): DriverPlugin = new QueryAsyncProfilerDriverPlugin
+ override def driverPlugin(): DriverPlugin = new DiagnoseDriverPlugin
- override def executorPlugin(): ExecutorPlugin = new AsyncProfilerExecutorPlugin
+ override def executorPlugin(): ExecutorPlugin = new DiagnoseExecutorPlugin
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala
index 1f6ad0ebba..2ebf1d4826 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala
@@ -18,28 +18,28 @@
package org.apache.spark.sql
-import java.io._
-import java.net.URI
-import java.nio.file.Paths
-
-import scala.collection.JavaConverters._
-
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.kylin.common.{KapConfig, KylinConfig}
import org.apache.kylin.common.util.{HadoopUtil, Unsafe}
+import org.apache.kylin.common.{KapConfig, KylinConfig}
import org.apache.kylin.metadata.query.BigQueryThresholdUpdater
+import org.apache.kylin.query.plugin.asyncprofiler.QueryAsyncProfilerSparkPlugin
+import org.apache.kylin.query.plugin.diagnose.DiagnoseSparkPlugin
import org.apache.kylin.query.util.ExtractFactory
-import org.springframework.expression.common.TemplateParserContext
-import org.springframework.expression.spel.standard.SpelExpressionParser
-
-import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql.SparkSession.Builder
-import org.apache.spark.sql.internal.{SessionState, SharedState, SQLConf, StaticSQLConf}
+import org.apache.spark.sql.internal.{SQLConf, SessionState, SharedState, StaticSQLConf}
import org.apache.spark.sql.udf.UdfManager
import org.apache.spark.util.{KylinReflectUtils, Utils}
+import org.apache.spark.{SparkConf, SparkContext}
+import org.springframework.expression.common.TemplateParserContext
+import org.springframework.expression.spel.standard.SpelExpressionParser
+
+import java.io._
+import java.net.URI
+import java.nio.file.Paths
+import scala.collection.JavaConverters._
class KylinSession(
@transient val sc: SparkContext,
@@ -112,6 +112,7 @@ class KylinSession(
object KylinSession extends Logging {
val NORMAL_FAIR_SCHEDULER_FILE_NAME: String = "/fairscheduler.xml"
val QUERY_LIMIT_FAIR_SCHEDULER_FILE_NAME: String = "/query-limit-fair-scheduler.xml"
+ val SPARK_PLUGINS_KEY = "spark.plugins"
implicit class KylinBuilder(builder: Builder) {
var queryCluster: Boolean = true
@@ -328,16 +329,30 @@ object KylinSession extends Logging {
}
}
+ checkAndSetSparkPlugins(sparkConf)
+
+ sparkConf
+ }
+
+ def checkAndSetSparkPlugins(sparkConf: SparkConf): Unit = {
+ // Sparder diagnose plugin
+ if (kapConfig.getKylinConfig.queryDiagnoseEnable()) {
+ addSparkPlugin(sparkConf, classOf[DiagnoseSparkPlugin].getCanonicalName)
+ }
+
+ // Query profile plugin
if (kapConfig.getKylinConfig.asyncProfilingEnabled()) {
- val plugins = sparkConf.get("spark.plugins", "")
- if (plugins.isEmpty) {
- sparkConf.set("spark.plugins", "org.apache.kylin.query.asyncprofiler.QueryAsyncProfilerSparkPlugin")
- } else {
- sparkConf.set("spark.plugins", "org.apache.kylin.query.asyncprofiler.QueryAsyncProfilerSparkPlugin," + plugins)
- }
+ addSparkPlugin(sparkConf, classOf[QueryAsyncProfilerSparkPlugin].getCanonicalName)
}
+ }
- sparkConf
+ def addSparkPlugin(sparkConf: SparkConf, pluginName: String): Unit = {
+ val plugins = sparkConf.get(SPARK_PLUGINS_KEY, "")
+ if (plugins.isEmpty) {
+ sparkConf.set(SPARK_PLUGINS_KEY, pluginName)
+ } else {
+ sparkConf.set(SPARK_PLUGINS_KEY, pluginName + "," + plugins)
+ }
}
def buildCluster(): KylinBuilder = {
@@ -411,7 +426,8 @@ object KylinSession extends Logging {
val parser = new SpelExpressionParser()
val parserCtx = new TemplateParserContext()
while ( {
- templateLine = fileReader.readLine(); templateLine != null
+ templateLine = fileReader.readLine();
+ templateLine != null
}) {
processedLine = parser.parseExpression(templateLine, parserCtx).getValue(params, classOf[String]) + "\r\n"
fileWriter.write(processedLine)
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
index 59ce599fce..cfb469d547 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
@@ -346,5 +346,8 @@ object SparderEnv extends Logging {
configuration
}
-
+ // Return the list of currently active executors
+ def getActiveExecutorIds(): Seq[String] = {
+ getSparkSession.sparkContext.getExecutorIds()
+ }
}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/SparkPluginWithMeta.scala
similarity index 94%
rename from src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala
rename to src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/SparkPluginWithMeta.scala
index 9db5d72000..064fb4516f 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/SparkPluginWithMeta.scala
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin
import org.apache.kylin.common.util.NLocalFileMetadataTestCase
import org.apache.spark.{SparkContext, SparkFunSuite}
@@ -24,7 +24,7 @@ import org.scalatest.BeforeAndAfterAll
import java.io.File
-trait AsyncPluginWithMeta extends SparkFunSuite with BeforeAndAfterAll {
+trait SparkPluginWithMeta extends SparkFunSuite with BeforeAndAfterAll {
@transient var sc: SparkContext = _
protected val ut_meta = "../examples/test_case_data/localmeta"
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/AsyncProfilingTest.scala
similarity index 94%
rename from src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
rename to src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/AsyncProfilingTest.scala
index 33f497ee33..81188e5aef 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/AsyncProfilingTest.scala
@@ -16,9 +16,10 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.asyncprofiler
import org.apache.kylin.common.KylinConfig
+import org.apache.kylin.query.plugin.SparkPluginWithMeta
import org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin
import org.apache.spark.launcher.SparkLauncher
import org.apache.spark.{SparkConf, SparkContext}
@@ -26,7 +27,7 @@ import org.mockito.Mockito.mock
import java.io.{File, OutputStream}
-class AsyncProfilingTest extends AsyncPluginWithMeta {
+class AsyncProfilingTest extends SparkPluginWithMeta {
val sparkPluginName: String = classOf[BuildAsyncProfilerSparkPlugin].getName
val flagFileDir: String = System.getProperty("java.io.tmpdir") + "default/jobStepId/"
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala
similarity index 96%
rename from src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala
rename to src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala
index 1ecbbe6a23..d0e21562cf 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryAsyncProfilerSparkPluginTest.scala
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.asyncprofiler
import org.apache.kylin.common.asyncprofiler.AsyncProfilerExecutorPlugin
import org.junit.Assert
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryProfilerDriverPluginTest.scala
similarity index 92%
rename from src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
rename to src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryProfilerDriverPluginTest.scala
index 44c564b3ba..74db3cc3a1 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/asyncprofiler/QueryProfilerDriverPluginTest.scala
@@ -16,13 +16,14 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.query.plugin.asyncprofiler
+import org.apache.kylin.query.plugin.SparkPluginWithMeta
import org.apache.spark.launcher.SparkLauncher
import org.apache.spark.{SparkConf, SparkContext}
import org.junit.Assert
-class QueryAsyncProfilerDriverPluginTest extends AsyncPluginWithMeta {
+class QueryAsyncProfilerDriverPluginTest extends SparkPluginWithMeta {
val sparkPluginName: String = classOf[QueryAsyncProfilerSparkPlugin].getName
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPluginTest.scala
new file mode 100644
index 0000000000..c725c3f83b
--- /dev/null
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnoseExecutorPluginTest.scala
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.plugin.diagnose
+
+import com.codahale.metrics.MetricRegistry
+import org.apache.kylin.query.plugin.SparkPluginWithMeta
+import org.apache.spark.api.plugin.PluginContext
+import org.apache.spark.launcher.SparkLauncher
+import org.apache.spark.resource.ResourceInformation
+import org.apache.spark.{SparkConf, SparkContext}
+import org.junit.Assert
+
+import java.nio.file.Files
+import java.util
+
+class DiagnoseExecutorPluginTest extends SparkPluginWithMeta {
+
+ val sparkPluginName: String = classOf[DiagnoseSparkPlugin].getName
+ val executorPluginTest = new DiagnoseExecutorPlugin()
+ val mockPluginCtx = new MockPluginCtx
+ val tempContainerDir = Files.createTempDirectory("PluginContainerTest")
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ val conf = new SparkConf()
+ .setAppName(getClass.getName)
+ .set(SparkLauncher.SPARK_MASTER, "local[1]")
+ .set("spark.plugins", sparkPluginName)
+ sc = new SparkContext(conf)
+
+ mockPluginCtx.sparkConf = sc.getConf
+ mockPluginCtx.hdfsDir = tempContainerDir.toString
+ mockPluginCtx.mockId = "mockId"
+
+ executorPluginTest.setCtx(mockPluginCtx)
+ executorPluginTest.setContainerDir(tempContainerDir.toFile)
+ }
+
+ override def afterAll(): Unit = {
+ super.afterAll()
+ mockPluginCtx.clear()
+ }
+
+
+ test("Test executor plugin") {
+ val filePath = Files.createTempFile(tempContainerDir, "gc", "log")
+ Assert.assertTrue(filePath.toFile.exists())
+
+ mockPluginCtx.message = DiagnoseConstant.COLLECT
+ executorPluginTest.checkAndDiagnose()
+
+ mockPluginCtx.message = DiagnoseConstant.NOP
+ executorPluginTest.checkAndDiagnose()
+
+ mockPluginCtx.message = DiagnoseConstant.EMPTY
+ executorPluginTest.checkAndDiagnose()
+ }
+}
+
+class MockPluginCtx() extends PluginContext {
+ var message: String = _
+ var mockId: String = _
+ var hdfsDir: String = _
+ var sparkConf: SparkConf = _
+
+ override def ask(input: Any): String = {
+ if (DiagnoseConstant.HDFSDIR.equals(input)) {
+ hdfsDir
+ } else {
+ message
+ }
+ }
+
+ override def executorID(): String = mockId
+
+ override def conf(): SparkConf = sparkConf
+
+ override def metricRegistry(): MetricRegistry = null
+
+ override def hostname(): String = "MockHostname"
+
+ override def resources(): util.Map[String, ResourceInformation] = null
+
+ override def send(message: Any): Unit = {}
+
+ def clear(): Unit = {
+ message = null
+ mockId = null
+ hdfsDir = null
+ sparkConf = null
+ }
+
+}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnosePluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnosePluginTest.scala
new file mode 100644
index 0000000000..75df31685e
--- /dev/null
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/plugin/diagnose/DiagnosePluginTest.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.plugin.diagnose
+
+import org.apache.kylin.common.KylinConfig
+import org.apache.kylin.query.plugin.SparkPluginWithMeta
+import org.apache.spark.launcher.SparkLauncher
+import org.apache.spark.{SparkConf, SparkContext}
+import org.junit.Assert
+
+class DiagnosePluginTest extends SparkPluginWithMeta {
+
+ val sparkPluginName: String = classOf[DiagnoseSparkPlugin].getName
+ val diagPlugin = new DiagnoseSparkPlugin
+ val driverPluginTest = new DiagnoseDriverPlugin()
+ val executorPluginTest = new DiagnoseExecutorPlugin()
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ val conf = new SparkConf()
+ .setAppName(getClass.getName)
+ .set(SparkLauncher.SPARK_MASTER, "local[1]")
+ .set("spark.plugins", sparkPluginName)
+ sc = new SparkContext(conf)
+ }
+
+
+ test("Test trigger gc collection") {
+ // get gc success
+ DiagnoseHelper.collectSparderExecutorGc()
+ Assert.assertEquals(DiagnoseConstant.STATE_WAIT, DiagnoseHelper.state)
+
+ // get gc failed
+ DiagnoseHelper.resultCollectionTimeout = 100L
+ DiagnoseHelper.activeExecutorCount = 1
+ DiagnoseHelper.collectSparderExecutorGc()
+ Assert.assertEquals(DiagnoseConstant.STATE_WAIT, DiagnoseHelper.state)
+ }
+
+
+ test("Test driver plugin") {
+ // NEXTCMD
+ DiagnoseHelper.setState(DiagnoseConstant.STATE_WAIT)
+ var reply = driverPluginTest.receive(DiagnoseConstant.NEXTCMD)
+ Assert.assertEquals(DiagnoseConstant.NOP, reply)
+
+ DiagnoseHelper.setState(DiagnoseConstant.STATE_COLLECT)
+ reply = driverPluginTest.receive(DiagnoseConstant.NEXTCMD)
+ Assert.assertEquals(DiagnoseConstant.COLLECT, reply)
+
+ // SENDRESULT
+ DiagnoseHelper.initGcCount(2)
+ reply = driverPluginTest.receive(DiagnoseConstant.SENDRESULT)
+ Assert.assertEquals(DiagnoseConstant.EMPTY, reply)
+ Assert.assertEquals(1, DiagnoseHelper.gcResult.getCount)
+
+ // HDFSDIR
+ reply = driverPluginTest.receive(DiagnoseConstant.HDFSDIR)
+ Assert.assertEquals(KylinConfig.getInstanceFromEnv.getHdfsWorkingDirectory, reply)
+
+ // Other
+ reply = driverPluginTest.receive("Other")
+ Assert.assertEquals(DiagnoseConstant.EMPTY, reply)
+ }
+
+}
[kylin] 02/38: KYLIN-5521 fix LeftOrInner Join
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 54940c9a9dced70a5d64fad376acdccc519b7400
Author: Jiale He <ji...@gmail.com>
AuthorDate: Fri Feb 17 16:11:38 2023 +0800
KYLIN-5521 fix LeftOrInner Join
---
.../src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
index b6342fd476..67f8e26bfc 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
@@ -355,7 +355,7 @@ public class JoinsGraph implements Serializable {
}
for (Edge targetEdge : edgeList) {
if (!edge.equals(targetEdge) && fkSide.equals(targetEdge.pkSide())
- && !targetEdge.isLeftOrInnerJoin() && targetEdge.isLeftJoin()) {
+ && !targetEdge.isLeftOrInnerJoin()) {
setJoinToLeftOrInner(targetEdge.join);
normalize();
}
[kylin] 34/38: [DIRTY] release kyspark 4.6.6.0
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e0599cbaed1e78a56f0a5fdaa99e29c3169de1c8
Author: Mingming Ge <7m...@gmail.com>
AuthorDate: Tue Mar 7 13:42:19 2023 +0800
[DIRTY] release kyspark 4.6.6.0
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index f7a6740b8f..00830ab3ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,7 +124,7 @@
<!-- Spark versions -->
<delta.version>1.2.1</delta.version>
- <spark.version>3.2.0-kylin-4.6.6.0-SNAPSHOT</spark.version>
+ <spark.version>3.2.0-kylin-4.6.6.0</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
[kylin] 18/38: KYLIN-5531 remove redundancy code
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e39b34cb4667430134d6054123acf34eb9992daf
Author: jlf <lo...@kyligence.io>
AuthorDate: Fri Feb 24 22:08:06 2023 +0800
KYLIN-5531 remove redundancy code
---
.../kylin/rest/service/ModelServiceBuildTest.java | 9 +-
.../engine/spark/builder/PartitionFlatTable.scala | 69 ---
.../engine/spark/builder/SegmentFlatTable.scala | 683 ---------------------
.../engine/spark/job/RDPartitionBuildExec.scala | 35 +-
.../engine/spark/job/RDSegmentBuildExec.scala | 32 +-
.../kylin/engine/spark/job/RDSegmentBuildJob.java | 13 +-
.../engine/spark/job/stage/build/BuildStage.scala | 5 +
.../job/stage/build/FlatTableAndDictBase.scala | 99 ++-
.../stage/build/MaterializedFactTableView.scala | 24 +-
.../partition/PartitionFlatTableAndDictBase.scala | 22 +
.../PartitionMaterializedFactTableView.scala | 30 +-
.../spark/smarter/IndexDependencyParser.scala | 16 +-
.../spark/builder/TestDimensionTableStat.scala | 13 +-
.../engine/spark/builder/TestFlatTable.scala} | 41 +-
.../engine/spark/builder/TestInferFilters.scala | 18 +-
.../spark/builder/TestSegmentFlatTable.scala | 35 +-
.../engine/spark/job/TestRDSegmentBuildExec.scala | 111 ++++
.../PartitionFlatTableAndDictBaseTest.scala | 88 +++
18 files changed, 394 insertions(+), 949 deletions(-)
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
index cb2896da8f..78ba0133e9 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.service;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CONCURRENT_SUBMIT_LIMIT;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_FAIL;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_ABANDON;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_DUPLICATE;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_LOCKED;
@@ -505,7 +504,6 @@ public class ModelServiceBuildTest extends SourceTestCase {
}
@Test
- @Ignore("Sorry, I don't know")
public void testBuildSegmentsManually_NoPartition_FullSegExisted() throws Exception {
val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
val project = "default";
@@ -527,13 +525,14 @@ public class ModelServiceBuildTest extends SourceTestCase {
modelService.updateDataModelSemantic(project, request);
try {
modelBuildService.buildSegmentsManually("default", "89af4ee2-2cdb-4b07-b39e-4c29856309aa", "", "");
+ Assert.fail();
} catch (TransactionException exception) {
Assert.assertTrue(exception.getCause() instanceof KylinException);
- Assert.assertEquals(JOB_CREATE_CHECK_FAIL.getErrorMsg().getLocalizedString(),
- exception.getCause().getMessage());
+ Assert.assertEquals(SEGMENT_STATUS.getErrorMsg().getCodeString(),
+ ((KylinException) exception.getCause()).getErrorCode().getCodeString());
}
val executables = getRunningExecutables(project, modelId);
- Assert.assertEquals(2, executables.size());
+ Assert.assertEquals(1, executables.size());
Assert.assertTrue(((NSparkCubingJob) executables.get(0)).getHandler() instanceof ExecutableAddCuboidHandler);
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/PartitionFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/PartitionFlatTable.scala
deleted file mode 100644
index 76c386d8c2..0000000000
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/PartitionFlatTable.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.spark.builder
-
-import org.apache.kylin.engine.spark.model.PartitionFlatTableDesc
-import org.apache.commons.lang3.StringUtils
-import org.apache.spark.sql.{Dataset, Row, SparkSession}
-
-import java.util.Objects
-import java.{lang, util}
-import scala.collection.JavaConverters._
-
-class PartitionFlatTable(private val sparkSession: SparkSession, //
- private val tableDesc: PartitionFlatTableDesc) extends SegmentFlatTable(sparkSession, tableDesc) {
-
- override protected def applyPartitionDesc(originDS: Dataset[Row]): Dataset[Row] = {
- // Multi level partition.
- val descMLP = dataModel.getMultiPartitionDesc
- require(Objects.nonNull(descMLP))
- // Date range partition.
- val descDRP = dataModel.getPartitionDesc
- val condition = descMLP.getPartitionConditionBuilder
- .buildMultiPartitionCondition(descDRP, descMLP, //
- new util.LinkedList[lang.Long](tableDesc.getPartitions), null, segmentRange)
- if (StringUtils.isBlank(condition)) {
- logInfo(s"Segment $segmentId no available partition condition.")
- return originDS
- }
- logInfo(s"Segment $segmentId apply partition condition $condition.")
- originDS.where(condition)
- }
-
- def getPartitionDS(partitionId: Long): Dataset[Row] = {
- val columnIds = tableDesc.getColumnIds.asScala
- val columnName2Id = tableDesc.getColumns //
- .asScala //
- .map(column => column.getIdentity) //
- .zip(columnIds) //
- val column2IdMap = columnName2Id.toMap
-
- val partitionColumnIds = dataModel.getMultiPartitionDesc.getColumnRefs.asScala //
- .map(_.getIdentity).map(x => column2IdMap.apply(x))
- val values = dataModel.getMultiPartitionDesc.getPartitionInfo(partitionId).getValues.toSeq
-
- val converted = partitionColumnIds.zip(values).map { case (k, v) =>
- s"`$k` = '$v'"
- }.mkString(" and ")
-
- logInfo(s"Segment $segmentId single partition condition: $converted")
- FLAT_TABLE.where(converted)
- }
-
-}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
deleted file mode 100644
index e6e41fd9b9..0000000000
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
+++ /dev/null
@@ -1,683 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.spark.builder
-
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-import java.util.{Locale, Objects, Timer, TimerTask}
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.kylin.common.constant.LogConstant
-import org.apache.kylin.common.logging.SetLogCategory
-import org.apache.kylin.common.util.HadoopUtil
-import org.apache.kylin.common.{CustomUtils, KapConfig, KylinConfig}
-import org.apache.kylin.engine.spark.builder.DFBuilderHelper._
-import org.apache.kylin.engine.spark.job.NSparkCubingUtil._
-import org.apache.kylin.engine.spark.job.{FiltersUtil, TableMetaManager}
-import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
-import org.apache.kylin.engine.spark.utils.LogEx
-import org.apache.kylin.engine.spark.utils.SparkDataSource._
-import org.apache.kylin.metadata.cube.model.NDataSegment
-import org.apache.kylin.metadata.model._
-import org.apache.kylin.query.util.PushDownUtil
-import org.apache.spark.sql._
-import org.apache.spark.sql.functions.{col, expr}
-import org.apache.spark.sql.manager.SparderLookupManager
-import org.apache.spark.sql.types.StructField
-import org.apache.spark.sql.util.SparderTypeUtil
-import org.apache.spark.utils.ProxyThreadUtils
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-import scala.collection.parallel.ForkJoinTaskSupport
-import scala.concurrent.duration.{Duration, MILLISECONDS}
-import scala.concurrent.forkjoin.ForkJoinPool
-import scala.util.{Failure, Success, Try}
-
-import com.google.common.collect.Sets
-
-class SegmentFlatTable(private val sparkSession: SparkSession, //
- private val tableDesc: SegmentFlatTableDesc) extends LogEx {
-
- import SegmentFlatTable._
-
- protected final val project = tableDesc.getProject
- protected final val spanningTree = tableDesc.getSpanningTree
- protected final val dataSegment = tableDesc.getDataSegment
- protected final val dataModel = tableDesc.getDataModel
- protected final val indexPlan = tableDesc.getIndexPlan
- protected final val segmentRange = tableDesc.getSegmentRange
- protected lazy final val flatTablePath = tableDesc.getFlatTablePath
- protected lazy final val factTableViewPath = tableDesc.getFactTableViewPath
- protected lazy final val workingDir = tableDesc.getWorkingDir
- protected lazy final val sampleRowCount = tableDesc.getSampleRowCount
-
- protected final val segmentId = dataSegment.getId
-
- protected lazy final val FLAT_TABLE = generateFlatTable()
- // flat-table without dict columns
- private lazy final val FLAT_TABLE_PART = generateFlatTablePart()
-
- protected val rootFactTable: TableRef = dataModel.getRootFactTable
-
- // Flat table.
- private lazy val shouldPersistFT = tableDesc.shouldPersistFlatTable()
- private lazy val isFTReady = dataSegment.isFlatTableReady && tableDesc.buildFilesSeparationPathExists(flatTablePath)
-
- // Fact table view.
- private lazy val isFTV = rootFactTable.getTableDesc.isView
- private lazy val shouldPersistFTV = tableDesc.shouldPersistView()
- private lazy val isFTVReady = dataSegment.isFactViewReady && HadoopUtil.getWorkingFileSystem.exists(factTableViewPath)
-
- private lazy val needJoin = {
- val join = tableDesc.shouldJoinLookupTables
- logInfo(s"Segment $segmentId flat table need join: $join")
- join
- }
-
- protected lazy val factTableDS: Dataset[Row] = newFactTableDS()
- private lazy val fastFactTableDS = newFastFactTableDS()
-
- // By design, COMPUTED-COLUMN could only be defined on fact table.
- protected lazy val factTableCCs: Set[TblColRef] = rootFactTable.getColumns.asScala
- .filter(_.getColumnDesc.isComputedColumn)
- .toSet
-
- def getFlatTablePartDS: Dataset[Row] = {
- FLAT_TABLE_PART
- }
-
- def getFlatTableDS: Dataset[Row] = {
- FLAT_TABLE
- }
-
- def gatherStatistics(): Statistics = {
- val stepDesc = s"Segment $segmentId collect flat table statistics."
- logInfo(stepDesc)
- sparkSession.sparkContext.setJobDescription(stepDesc)
- val statistics = gatherStatistics(FLAT_TABLE)
- logInfo(s"Segment $segmentId collect flat table statistics $statistics.")
- sparkSession.sparkContext.setJobDescription(null)
- statistics
- }
-
- protected def generateFlatTablePart(): Dataset[Row] = {
- val recoveredDS = tryRecoverFTDS()
- if (recoveredDS.nonEmpty) {
- return recoveredDS.get
- }
- var flatTableDS = if (needJoin) {
- val lookupTableDSMap = generateLookupTables()
- if (inferFiltersEnabled) {
- FiltersUtil.initFilters(tableDesc, lookupTableDSMap)
- }
- val jointDS = joinFactTableWithLookupTables(fastFactTableDS, lookupTableDSMap, dataModel, sparkSession)
- concatCCs(jointDS, factTableCCs)
- } else {
- concatCCs(fastFactTableDS, factTableCCs)
- }
- flatTableDS = applyFilterCondition(flatTableDS)
- changeSchemeToColumnId(flatTableDS, tableDesc)
- }
-
- protected def generateFlatTable(): Dataset[Row] = {
- val recoveredDS = tryRecoverFTDS()
- if (recoveredDS.nonEmpty) {
- return recoveredDS.get
- }
-
- /**
- * If need to build and encode dict columns, then
- * 1. try best to build in fact-table.
- * 2. try best to build in lookup-tables (without cc dict).
- * 3. try to build in fact-table.
- *
- * CC in lookup-tables MUST be built in flat-table.
- */
- val (dictCols, encodeCols, dictColsWithoutCc, encodeColsWithoutCc) = prepareForDict()
- val factTable = buildDictIfNeed(factTableDS, dictCols, encodeCols)
-
- var flatTable = if (needJoin) {
-
- val lookupTables = generateLookupTables()
- .map(lookupTableMap =>
- (lookupTableMap._1, buildDictIfNeed(lookupTableMap._2, dictColsWithoutCc, encodeColsWithoutCc)))
- if (lookupTables.nonEmpty) {
- generateLookupTableMeta(project, lookupTables)
- }
- if (inferFiltersEnabled) {
- FiltersUtil.initFilters(tableDesc, lookupTables)
- }
-
- val jointTable = joinFactTableWithLookupTables(factTable, lookupTables, dataModel, sparkSession)
- buildDictIfNeed(concatCCs(jointTable, factTableCCs),
- selectColumnsNotInTables(factTable, lookupTables.values.toSeq, dictCols),
- selectColumnsNotInTables(factTable, lookupTables.values.toSeq, encodeCols))
- } else {
- factTable
- }
-
- DFBuilderHelper.checkPointSegment(dataSegment, (copied: NDataSegment) => copied.setDictReady(true))
-
- flatTable = applyFilterCondition(flatTable)
- flatTable = changeSchemeToColumnId(flatTable, tableDesc)
- tryPersistFTDS(flatTable)
- }
-
- private def prepareForDict(): (Set[TblColRef], Set[TblColRef], Set[TblColRef], Set[TblColRef]) = {
- val dictCols = DictionaryBuilderHelper.extractTreeRelatedGlobalDictToBuild(dataSegment, spanningTree.getIndices).asScala.toSet
- val encodeCols = DictionaryBuilderHelper.extractTreeRelatedGlobalDicts(dataSegment, spanningTree.getIndices).asScala.toSet
- val dictColsWithoutCc = dictCols.filter(!_.getColumnDesc.isComputedColumn)
- val encodeColsWithoutCc = encodeCols.filter(!_.getColumnDesc.isComputedColumn)
- (dictCols, encodeCols, dictColsWithoutCc, encodeColsWithoutCc)
- }
-
- private def newFastFactTableDS(): Dataset[Row] = {
- val partDS = newPartitionedFTDS(needFast = true)
- fulfillDS(partDS, factTableCCs, rootFactTable)
- }
-
- private def newFactTableDS(): Dataset[Row] = {
- val partDS = newPartitionedFTDS()
- fulfillDS(partDS, factTableCCs, rootFactTable)
- }
-
- private def newPartitionedFTDS(needFast: Boolean = false): Dataset[Row] = {
- if (isFTVReady) {
- logInfo(s"Skip FACT-TABLE-VIEW segment $segmentId.")
- return sparkSession.read.parquet(factTableViewPath.toString)
- }
- val tableDS = newTableDS(rootFactTable)
- val partDS = applyPartitionDesc(tableDS)
- if (needFast || !isFTV) {
- return partDS
- }
- tryPersistFTVDS(partDS)
- }
-
- protected def generateLookupTables(): mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]] = {
- val ret = mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]]()
- val antiFlattenTableSet = mutable.Set[String]()
- dataModel.getJoinTables.asScala
- .filter(isTableToBuild)
- .foreach { joinDesc =>
- val fkTableRef = joinDesc.getJoin.getFKSide
- if (fkTableRef == null) {
- throw new IllegalArgumentException("FK table cannot be null")
- }
- val fkTable = fkTableRef.getTableDesc.getIdentity
- if (!joinDesc.isFlattenable || antiFlattenTableSet.contains(fkTable)) {
- antiFlattenTableSet.add(joinDesc.getTable)
- }
- if (joinDesc.isFlattenable && !antiFlattenTableSet.contains(joinDesc.getTable)) {
- val tableRef = joinDesc.getTableRef
- val tableDS = newTableDS(tableRef)
- ret.put(joinDesc, fulfillDS(tableDS, Set.empty, tableRef))
- }
- }
- ret
- }
-
- private def isTableToBuild(joinDesc: JoinTableDesc): Boolean = {
- !tableDesc.isPartialBuild || (tableDesc.isPartialBuild && tableDesc.getRelatedTables.contains(joinDesc.getAlias))
- }
-
- protected def applyPartitionDesc(originDS: Dataset[Row]): Dataset[Row] = {
- // Date range partition.
- val descDRP = dataModel.getPartitionDesc
- if (Objects.isNull(descDRP) //
- || Objects.isNull(descDRP.getPartitionDateColumn) //
- || Objects.isNull(segmentRange) //
- || segmentRange.isInfinite) {
- logInfo(s"No available PARTITION-CONDITION segment $segmentId")
- return originDS
- }
-
- val condition = descDRP.getPartitionConditionBuilder //
- .buildDateRangeCondition(descDRP, null, segmentRange)
- logInfo(s"Apply PARTITION-CONDITION $condition segment $segmentId")
- originDS.where(condition)
- }
-
- private def applyFilterCondition(originDS: Dataset[Row]): Dataset[Row] = {
- if (StringUtils.isBlank(dataModel.getFilterCondition)) {
- logInfo(s"No available FILTER-CONDITION segment $segmentId")
- return originDS
- }
- val expression = PushDownUtil.massageExpression(dataModel, project, dataModel.getFilterCondition, null)
- val converted = replaceDot(expression, dataModel)
- val condition = s" (1=1) AND ($converted)"
- logInfo(s"Apply FILTER-CONDITION: $condition segment $segmentId")
- originDS.where(condition)
- }
-
- private def tryPersistFTVDS(tableDS: Dataset[Row]): Dataset[Row] = {
- if (!shouldPersistFTV) {
- return tableDS
- }
- logInfo(s"Persist FACT-TABLE-VIEW $factTableViewPath")
- sparkSession.sparkContext.setJobDescription("Persist FACT-TABLE-VIEW.")
- tableDS.write.mode(SaveMode.Overwrite).parquet(factTableViewPath.toString)
- // Checkpoint fact table view.
- DFBuilderHelper.checkPointSegment(dataSegment, (copied: NDataSegment) => copied.setFactViewReady(true))
- val newDS = sparkSession.read.parquet(factTableViewPath.toString)
- sparkSession.sparkContext.setJobDescription(null)
- newDS
- }
-
- private def tryPersistFTDS(tableDS: Dataset[Row]): Dataset[Row] = {
- if (!shouldPersistFT) {
- return tableDS
- }
- if (tableDS.schema.isEmpty) {
- logInfo("No available flat table schema.")
- return tableDS
- }
- logInfo(s"Segment $segmentId persist flat table: $flatTablePath")
- sparkSession.sparkContext.setJobDescription(s"Segment $segmentId persist flat table.")
- val coalescePartitionNum = tableDesc.getFlatTableCoalescePartitionNum
- if (coalescePartitionNum > 0) {
- logInfo(s"Segment $segmentId flat table coalesce partition num $coalescePartitionNum")
- tableDS.coalesce(coalescePartitionNum) //
- .write.mode(SaveMode.Overwrite).parquet(flatTablePath.toString)
- } else {
- tableDS.write.mode(SaveMode.Overwrite).parquet(flatTablePath.toString)
- }
- DFBuilderHelper.checkPointSegment(dataSegment, (copied: NDataSegment) => {
- copied.setFlatTableReady(true)
- if (dataSegment.isFlatTableReady) {
- // KE-14714 if flat table is updated, there might be some data inconsistency across indexes
- copied.setStatus(SegmentStatusEnum.WARNING)
- }
- })
- val newDS = sparkSession.read.parquet(flatTablePath.toString)
- sparkSession.sparkContext.setJobDescription(null)
- newDS
- }
-
- private def tryRecoverFTDS(): Option[Dataset[Row]] = {
- if (tableDesc.isPartialBuild) {
- logInfo(s"Segment $segmentId no need reuse flat table for partial build.")
- return None
- } else if (!isFTReady) {
- logInfo(s"Segment $segmentId no available flat table.")
- return None
- }
- // +----------+---+---+---+---+-----------+-----------+
- // | 0| 2| 3| 4| 1|2_KE_ENCODE|4_KE_ENCODE|
- // +----------+---+---+---+---+-----------+-----------+
- val tableDS: DataFrame = Try(sparkSession.read.parquet(flatTablePath.toString)) match {
- case Success(df) => df
- case Failure(f) =>
- logInfo(s"Handled AnalysisException: Unable to infer schema for Parquet. Flat table path $flatTablePath is empty.", f)
- sparkSession.emptyDataFrame
- }
- // ([2_KE_ENCODE,4_KE_ENCODE], [0,1,2,3,4])
- val (coarseEncodes, noneEncodes) = tableDS.schema.map(sf => sf.name).partition(_.endsWith(ENCODE_SUFFIX))
- val encodes = coarseEncodes.map(_.stripSuffix(ENCODE_SUFFIX))
-
- val noneEncodesFieldMap: Map[String, StructField] = tableDS.schema.map(_.name)
- .zip(tableDS.schema.fields)
- .filter(p => noneEncodes.contains(p._1))
- .toMap
-
- val nones = tableDesc.getColumnIds.asScala //
- .zip(tableDesc.getColumns.asScala)
- .map(p => (String.valueOf(p._1), p._2)) //
- .filterNot(p => {
- val dataType = SparderTypeUtil.toSparkType(p._2.getType)
- noneEncodesFieldMap.contains(p._1) && (dataType == noneEncodesFieldMap(p._1).dataType)
- }) ++
- // [xx_KE_ENCODE]
- tableDesc.getMeasures.asScala //
- .map(DictionaryBuilderHelper.needGlobalDict) //
- .filter(Objects.nonNull) //
- .map(colRef => dataModel.getColumnIdByColumnName(colRef.getIdentity)) //
- .map(String.valueOf) //
- .filterNot(encodes.contains)
- .map(id => id + ENCODE_SUFFIX)
-
- if (nones.nonEmpty) {
- // The previous flat table missed some columns.
- // Flat table would be updated at afterwards step.
- logInfo(s"Segment $segmentId update flat table, columns should have been included " + //
- s"${nones.mkString("[", ",", "]")}")
- return None
- }
- // The previous flat table could be reusable.
- logInfo(s"Segment $segmentId skip build flat table.")
- Some(tableDS)
- }
-
- def newTableDS(tableRef: TableRef): Dataset[Row] = {
- // By design, why not try recovering from table snapshot.
- // If fact table is a view and its snapshot exists, that will benefit.
- logInfo(s"Load source table ${tableRef.getTableIdentity}")
- val tableDescCopy = tableRef.getTableDesc
- if (tableDescCopy.isTransactional || tableDescCopy.isRangePartition) {
- val model = tableRef.getModel
- if (Objects.nonNull(model)) {
- tableDescCopy.setPartitionDesc(model.getPartitionDesc)
- }
-
- if (Objects.nonNull(segmentRange) && Objects.nonNull(segmentRange.getStart) && Objects.nonNull(segmentRange.getEnd)) {
- sparkSession.table(tableDescCopy, segmentRange.getStart.toString, segmentRange.getEnd.toString).alias(tableRef.getAlias)
- } else {
- sparkSession.table(tableDescCopy).alias(tableRef.getAlias)
- }
- } else {
- sparkSession.table(tableDescCopy).alias(tableRef.getAlias)
- }
- }
-
- protected final def gatherStatistics(tableDS: Dataset[Row]): Statistics = {
- val totalRowCount = tableDS.count()
- if (!shouldPersistFT) {
- // By design, evaluating column bytes should be based on existed flat table.
- logInfo(s"Flat table not persisted, only compute row count.")
- return Statistics(totalRowCount, Map.empty[String, Long])
- }
- // zipWithIndex before filter
- val canonicalIndices = tableDS.columns //
- .zipWithIndex //
- .filterNot(_._1.endsWith(ENCODE_SUFFIX)) //
- .map { case (name, index) =>
- val canonical = tableDesc.getCanonicalName(Integer.parseInt(name))
- (canonical, index)
- }.filterNot(t => Objects.isNull(t._1))
- logInfo(s"CANONICAL INDICES ${canonicalIndices.mkString("[", ", ", "]")}")
- // By design, action-take is not sampling.
- val sampled = tableDS.take(sampleRowCount).flatMap(row => //
- canonicalIndices.map { case (canonical, index) => //
- val bytes = utf8Length(row.get(index))
- (canonical, bytes) //
- }).groupBy(_._1).mapValues(_.map(_._2).sum)
- val evaluated = evaluateColumnBytes(totalRowCount, sampled)
- Statistics(totalRowCount, evaluated)
- }
-
- private def evaluateColumnBytes(totalCount: Long, //
- sampled: Map[String, Long]): Map[String, Long] = {
- val multiple = if (totalCount < sampleRowCount) 1f else totalCount.toFloat / sampleRowCount
- sampled.mapValues(bytes => (bytes * multiple).toLong)
- }
-
- // Copied from DFChooser.
- private def utf8Length(value: Any): Long = {
- if (Objects.isNull(value)) {
- return 0L
- }
- var i = 0
- var bytes = 0L
- val sequence = value.toString
- while (i < sequence.length) {
- val c = sequence.charAt(i)
- if (c <= 0x7F) bytes += 1
- else if (c <= 0x7FF) bytes += 2
- else if (Character.isHighSurrogate(c)) {
- bytes += 4
- i += 1
- }
- else bytes += 3
- i += 1
- }
- bytes
- }
-
- // ====================================== Dividing line, till the bottom. ====================================== //
- // Historical debt.
- // Waiting for reconstruction.
-
- protected def buildDictIfNeed(table: Dataset[Row],
- dictCols: Set[TblColRef],
- encodeCols: Set[TblColRef]): Dataset[Row] = {
- if (dictCols.isEmpty && encodeCols.isEmpty) {
- return table
- }
- if (dataSegment.isDictReady) {
- logInfo(s"Skip DICTIONARY segment $segmentId")
- } else {
- // KE-32076 ensure at least one worker was registered before dictionary lock added.
- waitTillWorkerRegistered()
- buildDict(table, dictCols)
- }
- encodeColumn(table, encodeCols)
- }
-
- def waitTillWorkerRegistered(): Unit = {
- val cdl = new CountDownLatch(1)
- val timer = new Timer("worker-starvation-timer", true)
- timer.scheduleAtFixedRate(new TimerTask {
- override def run(): Unit = {
- if (sparkSession.sparkContext.statusTracker.getExecutorInfos.isEmpty) {
- logWarning("Ensure at least one worker has been registered before building dictionary.")
- } else {
- this.cancel()
- cdl.countDown()
- }
- }
- }, 0, TimeUnit.SECONDS.toMillis(20))
- cdl.await()
- timer.cancel()
- }
-
- private def concatCCs(table: Dataset[Row], computColumns: Set[TblColRef]): Dataset[Row] = {
- val matchedCols = selectColumnsInTable(table, computColumns)
- var tableWithCcs = table
- matchedCols.foreach(m =>
- tableWithCcs = tableWithCcs.withColumn(convertFromDot(m.getBackTickIdentity),
- expr(convertFromDot(m.getBackTickExp))))
- tableWithCcs
- }
-
- private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef]): Unit = {
- var matchedCols = selectColumnsInTable(ds, dictCols)
- if (dataSegment.getIndexPlan.isSkipEncodeIntegerFamilyEnabled) {
- matchedCols = matchedCols.filterNot(_.getType.isIntegerFamily)
- }
- val builder = new DFDictionaryBuilder(ds, dataSegment, sparkSession, Sets.newHashSet(matchedCols.asJavaCollection))
- builder.buildDictSet()
- }
-
- private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef]): Dataset[Row] = {
- val matchedCols = selectColumnsInTable(ds, encodeCols)
- var encodeDs = ds
- if (matchedCols.nonEmpty) {
- encodeDs = DFTableEncoder.encodeTable(ds, dataSegment, matchedCols.asJava)
- }
- encodeDs
- }
-
-}
-
-object SegmentFlatTable extends LogEx {
-
- import org.apache.kylin.engine.spark.job.NSparkCubingUtil._
-
- private val conf = KylinConfig.getInstanceFromEnv
- var inferFiltersEnabled: Boolean = conf.inferFiltersEnabled()
-
- def fulfillDS(originDS: Dataset[Row], cols: Set[TblColRef], tableRef: TableRef): Dataset[Row] = {
- // wrap computed columns, filter out valid columns
- val computedColumns = chooseSuitableCols(originDS, cols)
- // wrap alias
- val newDS = wrapAlias(originDS, tableRef.getAlias)
- val selectedColumns = newDS.schema.fields.map(tp => col(tp.name)) ++ computedColumns
- logInfo(s"Table SCHEMA ${tableRef.getTableIdentity} ${newDS.schema.treeString}")
- newDS.select(selectedColumns: _*)
- }
-
- def wrapAlias(originDS: Dataset[Row], alias: String): Dataset[Row] = {
- val newFields = originDS.schema.fields.map(f =>
- convertFromDot("`" + alias + "`" + "." + "`" + f.name + "`")).toSeq
- val newDS = originDS.toDF(newFields: _*)
- CustomUtils.tryWithResourceIgnore(new SetLogCategory(LogConstant.BUILD_CATEGORY)) {
- _ => logInfo(s"Wrap ALIAS ${originDS.schema.treeString} TO ${newDS.schema.treeString}")
- }
- newDS
- }
-
-
- def joinFactTableWithLookupTables(rootFactDataset: Dataset[Row],
- lookupTableDatasetMap: mutable.Map[JoinTableDesc, Dataset[Row]],
- model: NDataModel,
- ss: SparkSession): Dataset[Row] = {
- lookupTableDatasetMap.foldLeft(rootFactDataset)(
- (joinedDataset: Dataset[Row], tuple: (JoinTableDesc, Dataset[Row])) =>
- joinTableDataset(model.getRootFactTable.getTableDesc, tuple._1, joinedDataset, tuple._2, ss))
- }
-
- def joinTableDataset(rootFactDesc: TableDesc,
- lookupDesc: JoinTableDesc,
- rootFactDataset: Dataset[Row],
- lookupDataset: Dataset[Row],
- ss: SparkSession): Dataset[Row] = {
- var afterJoin = rootFactDataset
- val join = lookupDesc.getJoin
- if (join != null && !StringUtils.isEmpty(join.getType)) {
- val joinType = join.getType.toUpperCase(Locale.ROOT)
- val pk = join.getPrimaryKeyColumns
- val fk = join.getForeignKeyColumns
- if (pk.length != fk.length) {
- throw new RuntimeException(
- s"Invalid join condition of fact table: $rootFactDesc,fk: ${fk.mkString(",")}," +
- s" lookup table:$lookupDesc, pk: ${pk.mkString(",")}")
- }
- val equiConditionColPairs = fk.zip(pk).map(joinKey =>
- col(convertFromDot(joinKey._1.getBackTickIdentity))
- .equalTo(col(convertFromDot(joinKey._2.getBackTickIdentity))))
- CustomUtils.tryWithResourceIgnore(new SetLogCategory(LogConstant.BUILD_CATEGORY)) {
- _ => logInfo(s"Lookup table schema ${lookupDataset.schema.treeString}")
- }
-
- if (join.getNonEquiJoinCondition != null) {
- var condition = NonEquiJoinConditionBuilder.convert(join.getNonEquiJoinCondition)
- if (!equiConditionColPairs.isEmpty) {
- condition = condition && equiConditionColPairs.reduce(_ && _)
- }
- logInfo(s"Root table ${rootFactDesc.getIdentity}, join table ${lookupDesc.getAlias}, non-equi condition: ${condition.toString()}")
- afterJoin = afterJoin.join(lookupDataset, condition, joinType)
- } else {
- val condition = equiConditionColPairs.reduce(_ && _)
- logInfo(s"Root table ${rootFactDesc.getIdentity}, join table ${lookupDesc.getAlias}, condition: ${condition.toString()}")
- if (inferFiltersEnabled) {
- afterJoin = afterJoin.join(FiltersUtil.inferFilters(pk, lookupDataset), condition, joinType)
- } else {
- afterJoin = afterJoin.join(lookupDataset, condition, joinType)
- }
- }
- }
- afterJoin
- }
-
- def changeSchemeToColumnId(ds: Dataset[Row], tableDesc: SegmentFlatTableDesc): Dataset[Row] = {
- val structType = ds.schema
- val columnIds = tableDesc.getColumnIds.asScala
- val columnName2Id = tableDesc.getColumns
- .asScala
- .map(column => convertFromDot(column.getBackTickIdentity))
- .zip(columnIds)
- val columnName2IdMap = columnName2Id.toMap
- val encodeSeq = structType.filter(_.name.endsWith(ENCODE_SUFFIX)).map {
- tp =>
- val columnName = tp.name.stripSuffix(ENCODE_SUFFIX)
- val columnId = columnName2IdMap.apply(columnName)
- col(tp.name).alias(columnId.toString + ENCODE_SUFFIX)
- }
- val columns = columnName2Id.map(tp => expr("`" + tp._1 + "`").alias(tp._2.toString))
- logInfo(s"Select model column is ${columns.mkString(",")}")
- logInfo(s"Select model encoding column is ${encodeSeq.mkString(",")}")
- val selectedColumns = columns ++ encodeSeq
-
- logInfo(s"Select model all column is ${selectedColumns.mkString(",")}")
- ds.select(selectedColumns: _*)
- }
-
- private def generateLookupTableMeta(project: String,
- lookupTables: mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]]): Unit = {
- val config = KapConfig.getInstanceFromEnv
- if (config.isRecordSourceUsage) {
- lookupTables.keySet.foreach { joinTable =>
- val tableManager = NTableMetadataManager.getInstance(config.getKylinConfig, project)
- val table = tableManager.getOrCreateTableExt(joinTable.getTable)
- if (table.getTotalRows > 0) {
- TableMetaManager.putTableMeta(joinTable.getTable, 0, table.getTotalRows)
- logInfo(s"put meta table: ${joinTable.getTable}, count: ${table.getTotalRows}")
- }
- }
- }
- val noStatLookupTables = lookupTables.filterKeys(table => TableMetaManager.getTableMeta(table.getTable).isEmpty)
- if (config.getKylinConfig.isNeedCollectLookupTableInfo && noStatLookupTables.nonEmpty) {
- val lookupTablePar = noStatLookupTables.par
- lookupTablePar.tasksupport = new ForkJoinTaskSupport(new ForkJoinPool(lookupTablePar.size))
- lookupTablePar.foreach { case (joinTableDesc, dataset) =>
- val tableIdentity = joinTableDesc.getTable
- logTime(s"count $tableIdentity") {
- val maxTime = Duration(config.getKylinConfig.getCountLookupTableMaxTime, MILLISECONDS)
- val defaultCount = config.getKylinConfig.getLookupTableCountDefaultValue
- val rowCount = countTableInFiniteTimeOrDefault(dataset, tableIdentity, maxTime, defaultCount)
- TableMetaManager.putTableMeta(tableIdentity, 0L, rowCount)
- logInfo(s"put meta table: $tableIdentity , count: $rowCount")
- }
- }
- }
- }
-
- def countTableInFiniteTimeOrDefault(dataset: Dataset[Row], tableName: String,
- duration: Duration, defaultCount: Long): Long = {
- val countTask = dataset.rdd.countAsync()
- try {
- ProxyThreadUtils.awaitResult(countTask, duration)
- } catch {
- case e: Exception =>
- countTask.cancel()
- logInfo(s"$tableName count fail, and return defaultCount $defaultCount", e)
- defaultCount
- }
- }
-
- def replaceDot(original: String, model: NDataModel): String = {
- val sb = new StringBuilder(original)
-
- for (namedColumn <- model.getAllNamedColumns.asScala) {
- val colName = namedColumn.getAliasDotColumn.toLowerCase(Locale.ROOT)
- doReplaceDot(sb, colName, namedColumn.getAliasDotColumn)
-
- // try replacing quoted identifiers if any
- val quotedColName = colName.split('.').mkString("`", "`.`", "`")
- if (quotedColName.nonEmpty) {
- doReplaceDot(sb, quotedColName, namedColumn.getAliasDotColumn.split('.').mkString("`", "`.`", "`"))
- }
- }
- sb.toString()
- }
-
- private def doReplaceDot(sb: StringBuilder, namedCol: String, colAliasDotColumn: String): Unit = {
- var start = sb.toString.toLowerCase(Locale.ROOT).indexOf(namedCol)
- while (start != -1) {
- sb.replace(start,
- start + namedCol.length,
- "`" + convertFromDot(colAliasDotColumn) + "`")
- start = sb.toString.toLowerCase(Locale.ROOT)
- .indexOf(namedCol)
- }
- }
-
- case class Statistics(totalCount: Long, columnBytes: Map[String, Long])
-
-}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDPartitionBuildExec.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDPartitionBuildExec.scala
index d643cd6c05..6fe76fe447 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDPartitionBuildExec.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDPartitionBuildExec.scala
@@ -19,11 +19,11 @@
package org.apache.kylin.engine.spark.job
import io.kyligence.kap.guava20.shaded.common.collect.{Maps, Sets}
+import io.kyligence.kap.guava20.shaded.common.collect.Maps
import org.apache.hadoop.fs.Path
-import org.apache.kylin.engine.spark.builder.PartitionFlatTable
-import org.apache.kylin.engine.spark.model.PartitionFlatTableDesc
-import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree
-import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree.{PartitionTreeBuilder, PartitionTreeNode}
+import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.engine.spark.job.stage.build.partition.PartitionFlatTableAndDictBase
+import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree.PartitionTreeNode
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.spark.sql.SparderEnv
import org.apache.spark.sql.datasource.storage.StorageStoreUtils
@@ -32,31 +32,18 @@ import org.apache.spark.sql.hive.utils.ResourceDetectUtils
import java.io.IOException
import scala.collection.JavaConverters._
-class RDPartitionBuildExec(private val jobContext: RDSegmentBuildJob, //
- private val dataSegment: NDataSegment) extends RDSegmentBuildExec(jobContext, dataSegment) {
-
- private val newBuckets =
- jobContext.getReadOnlyBuckets.asScala.filter(_.getSegmentId.equals(segmentId)).toSeq
-
- protected final lazy val partitions = {
- val distincted = newBuckets.map(_.getPartitionId).distinct.sorted
- logInfo(s"Segment $segmentId partitions: ${distincted.mkString("[", ",", "]")}")
- scala.collection.JavaConverters.seqAsJavaList(distincted.map(java.lang.Long.valueOf))
- }
-
- private lazy val spanningTree = new PartitionSpanningTree(config, //
- new PartitionTreeBuilder(dataSegment, readOnlyLayouts, jobId, partitions, Sets.newHashSet(newBuckets.asJava)))
-
- private lazy val flatTableDesc = new PartitionFlatTableDesc(config, dataSegment, spanningTree, jobId, partitions)
-
- private lazy val flatTable = new PartitionFlatTable(sparkSession, flatTableDesc)
+class RDPartitionBuildExec(private val jobContext: SegmentJob, //
+ private val dataSegment: NDataSegment, private val buildParam: BuildParam)
+ extends PartitionFlatTableAndDictBase(jobContext, dataSegment, buildParam) {
+ protected final val rdSharedPath = jobContext.getRdSharedPath
@throws(classOf[IOException])
- override def detectResource(): Unit = {
+ def detectResource(): Unit = {
+ initFlatTableOnDetectResource()
val flatTableExecutions = if (spanningTree.fromFlatTable()) {
- Seq((-1L, Seq(flatTable.getFlatTablePartDS.queryExecution)))
+ Seq((-1L, Seq(getFlatTablePartDS.queryExecution)))
} else {
Seq.empty
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildExec.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildExec.scala
index a2c272dec6..bfaddaa11d 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildExec.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildExec.scala
@@ -20,12 +20,9 @@ package org.apache.kylin.engine.spark.job
import com.google.common.collect.Maps
import org.apache.hadoop.fs.Path
-import org.apache.kylin.engine.spark.builder.SegmentFlatTable
-import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
+import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.engine.spark.job.stage.build.FlatTableAndDictBase
import org.apache.kylin.metadata.cube.model.NDataSegment
-import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparderEnv
import org.apache.spark.sql.datasource.storage.StorageStoreUtils
import org.apache.spark.sql.hive.utils.ResourceDetectUtils
@@ -33,33 +30,20 @@ import org.apache.spark.sql.hive.utils.ResourceDetectUtils
import java.io.IOException
import scala.collection.JavaConverters._
-class RDSegmentBuildExec(private val jobContext: RDSegmentBuildJob, //
- private val dataSegment: NDataSegment) extends Logging {
+class RDSegmentBuildExec(private val jobContext: SegmentJob, //
+ private val dataSegment: NDataSegment, private val buildParam: BuildParam
+ )
+ extends FlatTableAndDictBase(jobContext, dataSegment, buildParam) {
// Resource detect segment build exec.
- // Needed variables from job context.
- protected final val jobId = jobContext.getJobId
- protected final val config = jobContext.getConfig
- protected final val dataflowId = jobContext.getDataflowId
- protected final val sparkSession = jobContext.getSparkSession
protected final val rdSharedPath = jobContext.getRdSharedPath
- protected final val readOnlyLayouts = jobContext.getReadOnlyLayouts
-
- // Needed variables from data segment.
- protected final val segmentId = dataSegment.getId
- protected final val project = dataSegment.getProject
-
- private lazy val spanningTree = new AdaptiveSpanningTree(config, new AdaptiveTreeBuilder(dataSegment, readOnlyLayouts))
-
- private lazy val flatTableDesc = new SegmentFlatTableDesc(config, dataSegment, spanningTree)
-
- private lazy val flatTable = new SegmentFlatTable(sparkSession, flatTableDesc)
@throws(classOf[IOException])
def detectResource(): Unit = {
+ initFlatTableOnDetectResource()
val flatTableExecutions = if (spanningTree.fromFlatTable()) {
- Seq((-1L, flatTable.getFlatTablePartDS.queryExecution))
+ Seq((-1L, getFlatTablePartDS.queryExecution))
} else {
Seq.empty
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java
index f4809375ab..2fbf4652dc 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java
@@ -24,9 +24,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.kylin.engine.spark.job.LogJobInfoUtils;
import org.apache.kylin.engine.spark.job.RDPartitionBuildExec;
import org.apache.kylin.engine.spark.job.RDSegmentBuildExec;
+import org.apache.kylin.engine.spark.job.ResourceDetect;
+import org.apache.kylin.engine.spark.job.SegmentJob;
+import org.apache.kylin.engine.spark.job.stage.BuildParam;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.spark.sql.hive.utils.ResourceDetectUtils;
+import lombok.val;
+
public class RDSegmentBuildJob extends SegmentJob implements ResourceDetect {
public static void main(String[] args) {
@@ -46,14 +51,16 @@ public class RDSegmentBuildJob extends SegmentJob implements ResourceDetect {
private void detectPartition() throws IOException {
for (NDataSegment dataSegment : readOnlySegments) {
- RDSegmentBuildExec exec = new RDPartitionBuildExec(this, dataSegment);
+ val buildParam = new BuildParam();
+ val exec = new RDPartitionBuildExec(this, dataSegment, buildParam);
exec.detectResource();
}
}
private void detect() throws IOException {
for (NDataSegment dataSegment : readOnlySegments) {
- RDSegmentBuildExec exec = new RDSegmentBuildExec(this, dataSegment);
+ val buildParam = new BuildParam();
+ val exec = new RDSegmentBuildExec(this, dataSegment, buildParam);
exec.detectResource();
}
}
@@ -65,6 +72,6 @@ public class RDSegmentBuildJob extends SegmentJob implements ResourceDetect {
private void writeCountDistinct() {
ResourceDetectUtils.write(new Path(rdSharedPath, ResourceDetectUtils.countDistinctSuffix()), //
- ResourceDetectUtils.findCountDistinctMeasure(getReadOnlyLayouts()));
+ ResourceDetectUtils.findCountDistinctMeasure(readOnlyLayouts));
}
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
index dba42cca96..2cfe5b7e2d 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
@@ -559,4 +559,9 @@ abstract class BuildStage(private val jobContext: SegmentJob,
// ----------------------------- Beta feature: Inferior Flat Table. ----------------------------- //
+ override def getStageName: String = ""
+
+ override def execute(): Unit = {
+ // parent class is empty
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index a8b28d283b..0599fc95ae 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -31,9 +31,12 @@ import org.apache.kylin.engine.spark.job.NSparkCubingUtil.convertFromDot
import org.apache.kylin.engine.spark.job.stage.{BuildParam, StageExec}
import org.apache.kylin.engine.spark.job.{FiltersUtil, SegmentJob, TableMetaManager}
import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
+import org.apache.kylin.engine.spark.smarter.IndexDependencyParser
import org.apache.kylin.engine.spark.model.planner.{CuboIdToLayoutUtils, FlatTableToCostUtils}
import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.kylin.engine.spark.utils.SparkDataSource._
+import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
+import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.cube.planner.CostBasePlannerUtils
import org.apache.kylin.metadata.model._
@@ -485,29 +488,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
total = tableSizeMap(tableName).longValue()
logInfo(s"Find $column's table $tableName count $total from cache")
} else {
- val catalogStatistics = TableMetaManager.getTableMeta(tableDesc.getTableName(column))
- if (catalogStatistics.isDefined) {
- total = catalogStatistics.get.rowCount.get.longValue()
- logInfo(s"Find $column's table $tableName count $total from catalog")
- } else {
- val tableMetadataDesc = tableMetadataManager.getTableDesc(tableDesc.getTableName(column))
- if (tableMetadataDesc != null) {
- val tableExtDesc = tableMetadataManager.getTableExtIfExists(tableMetadataDesc)
- if (tableExtDesc.getTotalRows > 0) {
- total = tableExtDesc.getTotalRows
- logInfo(s"Find $column's table $tableName count $total from table ext")
- } else if (tableMetadataDesc.getLastSnapshotPath != null) {
- val baseDir = KapConfig.getInstanceFromEnv.getMetadataWorkingDirectory
- val fs = HadoopUtil.getWorkingFileSystem
- val path = new Path(baseDir, tableMetadataDesc.getLastSnapshotPath)
- if (fs.exists(path)) {
- total = sparkSession.read.parquet(path.toString).count()
- logInfo(s"Calculate $column's table $tableName count $total " +
- s"from parquet ${tableMetadataDesc.getLastSnapshotPath}")
- }
- }
- }
- }
+ total = evaluateColumnTotalFromTableDesc(tableMetadataManager, totalCount, tableName, column)
}
}
} catch {
@@ -528,6 +509,29 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
}
}
+ def evaluateColumnTotalFromTableDesc(tableMetadataManager: NTableMetadataManager, totalCount: Long, //
+ tableName: String, column: String): Long = {
+ var total: Long = totalCount
+ val tableMetadataDesc = tableMetadataManager.getTableDesc(tableDesc.getTableName(column))
+ if (tableMetadataDesc != null) {
+ val tableExtDesc = tableMetadataManager.getTableExtIfExists(tableMetadataDesc)
+ if (tableExtDesc.getTotalRows > 0) {
+ total = tableExtDesc.getTotalRows
+ logInfo(s"Find $column's table $tableName count $total from table ext")
+ } else if (tableMetadataDesc.getLastSnapshotPath != null) {
+ val baseDir = KapConfig.getInstanceFromEnv.getMetadataWorkingDirectory
+ val fs = HadoopUtil.getWorkingFileSystem
+ val path = new Path(baseDir, tableMetadataDesc.getLastSnapshotPath)
+ if (fs.exists(path)) {
+ total = sparkSession.read.parquet(path.toString).count()
+ logInfo(s"Calculate $column's table $tableName count $total " +
+ s"from parquet ${tableMetadataDesc.getLastSnapshotPath}")
+ }
+ }
+ }
+ total
+ }
+
// Copied from DFChooser.
private def utf8Length(value: Any): Long = {
if (Objects.isNull(value)) {
@@ -648,6 +652,42 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
}
encodeDs
}
+
+ protected def initSpanningTree(): Unit = {
+ val spanTree = new AdaptiveSpanningTree(config, new AdaptiveTreeBuilder(dataSegment, readOnlyLayouts))
+ buildParam.setSpanningTree(spanTree)
+ }
+
+ protected def initFlatTableDesc(): Unit = {
+ val flatTableDesc: SegmentFlatTableDesc = if (jobContext.isPartialBuild) {
+ val parser = new IndexDependencyParser(dataModel)
+ val relatedTableAlias =
+ parser.getRelatedTablesAlias(jobContext.getReadOnlyLayouts)
+ new SegmentFlatTableDesc(config, dataSegment, spanningTree, relatedTableAlias)
+ } else {
+ new SegmentFlatTableDesc(config, dataSegment, spanningTree)
+ }
+ buildParam.setFlatTableDesc(flatTableDesc)
+ }
+
+ def initFactTable(): Unit = {
+ val factTableDS: Dataset[Row] = newFactTableDS()
+ buildParam.setFactTableDS(factTableDS)
+ val fastFactTableDS: Dataset[Row] = newFastFactTableDS()
+ buildParam.setFastFactTableDS(fastFactTableDS)
+ }
+
+ def materializedFactTableView(): Unit = {
+ initSpanningTree()
+ initFlatTableDesc()
+ initFactTable()
+ }
+
+ def initFlatTableOnDetectResource(): Unit = {
+ materializedFactTableView()
+ val flatTablePart: Dataset[Row] = generateFlatTablePart()
+ buildParam.setFlatTablePart(flatTablePart)
+ }
}
object FlatTableAndDictBase extends LogEx {
@@ -707,10 +747,7 @@ object FlatTableAndDictBase extends LogEx {
logInfo(s"Lookup table schema ${lookupDataset.schema.treeString}")
if (join.getNonEquiJoinCondition != null) {
- var condition = NonEquiJoinConditionBuilder.convert(join.getNonEquiJoinCondition)
- if (!equiConditionColPairs.isEmpty) {
- condition = condition && equiConditionColPairs.reduce(_ && _)
- }
+ val condition: Column = getCondition(join, equiConditionColPairs)
logInfo(s"Root table ${rootFactDesc.getIdentity}, join table ${lookupDesc.getAlias}, non-equi condition: ${condition.toString()}")
afterJoin = afterJoin.join(lookupDataset, condition, joinType)
} else {
@@ -726,6 +763,14 @@ object FlatTableAndDictBase extends LogEx {
afterJoin
}
+ def getCondition(join: JoinDesc, equiConditionColPairs: Array[Column]): Column = {
+ var condition = NonEquiJoinConditionBuilder.convert(join.getNonEquiJoinCondition)
+ if (!equiConditionColPairs.isEmpty) {
+ condition = condition && equiConditionColPairs.reduce(_ && _)
+ }
+ condition
+ }
+
def changeSchemeToColumnId(ds: Dataset[Row], tableDesc: SegmentFlatTableDesc): Dataset[Row] = {
val structType = ds.schema
val columnIds = tableDesc.getColumnIds.asScala
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
index 632ae0cd1c..19b5cd073c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
@@ -20,36 +20,14 @@ package org.apache.kylin.engine.spark.job.stage.build
import org.apache.kylin.engine.spark.job.SegmentJob
import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
-import org.apache.kylin.engine.spark.smarter.IndexDependencyParser
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
import org.apache.kylin.metadata.cube.model.NDataSegment
-import org.apache.spark.sql.{Dataset, Row}
class MaterializedFactTableView(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends FlatTableAndDictBase(jobContext, dataSegment, buildParam) {
override def execute(): Unit = {
logInfo(s"Build SEGMENT $segmentId")
- val spanTree = new AdaptiveSpanningTree(config, new AdaptiveTreeBuilder(dataSegment, readOnlyLayouts))
- buildParam.setSpanningTree(spanTree)
-
- val flatTableDesc: SegmentFlatTableDesc = if (jobContext.isPartialBuild) {
- val parser = new IndexDependencyParser(dataModel)
- val relatedTableAlias =
- parser.getRelatedTablesAlias(readOnlyLayouts)
- new SegmentFlatTableDesc(config, dataSegment, spanningTree, relatedTableAlias)
- } else {
- new SegmentFlatTableDesc(config, dataSegment, spanningTree)
- }
- buildParam.setFlatTableDesc(flatTableDesc)
-
- val factTableDS: Dataset[Row] = newFactTableDS()
- buildParam.setFactTableDS(factTableDS)
-
- val fastFactTableDS: Dataset[Row] = newFastFactTableDS()
- buildParam.setFastFactTableDS(fastFactTableDS)
+ materializedFactTableView()
if (buildParam.isSkipMaterializedFactTableView) {
onStageSkipped()
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBase.scala
index 4b9a7bdfc0..af1d8251c7 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBase.scala
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.spark.job.stage.build.partition
+import io.kyligence.kap.guava20.shaded.common.collect.Sets
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.engine.spark.builder.{DictionaryBuilderHelper, PartitionDictionaryBuilderHelper}
import org.apache.kylin.engine.spark.job.stage.BuildParam
@@ -25,6 +26,9 @@ import org.apache.kylin.engine.spark.job.stage.build.FlatTableAndDictBase
import org.apache.kylin.engine.spark.job.stage.build.FlatTableAndDictBase.Statistics
import org.apache.kylin.engine.spark.job.{PartitionExec, SegmentJob}
import org.apache.kylin.engine.spark.model.PartitionFlatTableDesc
+import org.apache.kylin.engine.spark.smarter.IndexDependencyParser
+import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree
+import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree.PartitionTreeBuilder
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.model.TblColRef
import org.apache.spark.sql.{Dataset, Row}
@@ -99,4 +103,22 @@ abstract class PartitionFlatTableAndDictBase(private val jobContext: SegmentJob,
val encodeColsWithoutCc = encodeCols.filter(!_.getColumnDesc.isComputedColumn)
(dictCols, encodeCols, dictColsWithoutCc, encodeColsWithoutCc)
}
+
+ override def initSpanningTree(): Unit = {
+ val spanTree = new PartitionSpanningTree(config, //
+ new PartitionTreeBuilder(dataSegment, readOnlyLayouts, jobId, partitions, Sets.newHashSet(newBuckets.asJava)))
+ buildParam.setPartitionSpanningTree(spanTree)
+ }
+
+ override def initFlatTableDesc(): Unit = {
+ val tableDesc = if (jobContext.isPartialBuild) {
+ val parser = new IndexDependencyParser(dataModel)
+ val relatedTableAlias =
+ parser.getRelatedTablesAlias(jobContext.getReadOnlyLayouts)
+ new PartitionFlatTableDesc(config, dataSegment, spanningTree, relatedTableAlias, jobId, partitions)
+ } else {
+ new PartitionFlatTableDesc(config, dataSegment, spanningTree, jobId, partitions)
+ }
+ buildParam.setTableDesc(tableDesc)
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionMaterializedFactTableView.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionMaterializedFactTableView.scala
index deb1c604bb..3f04461b15 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionMaterializedFactTableView.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionMaterializedFactTableView.scala
@@ -20,15 +20,7 @@ package org.apache.kylin.engine.spark.job.stage.build.partition
import org.apache.kylin.engine.spark.job.SegmentJob
import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.engine.spark.model.PartitionFlatTableDesc
-import org.apache.kylin.engine.spark.smarter.IndexDependencyParser
-import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree
-import org.apache.kylin.metadata.cube.cuboid.PartitionSpanningTree.PartitionTreeBuilder
import org.apache.kylin.metadata.cube.model.NDataSegment
-import org.apache.kylin.metadata.job.JobBucket
-import org.apache.spark.sql.{Dataset, Row}
-
-import java.util.stream.Collectors
class PartitionMaterializedFactTableView(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends PartitionFlatTableAndDictBase(jobContext, dataSegment, buildParam) {
@@ -36,27 +28,7 @@ class PartitionMaterializedFactTableView(jobContext: SegmentJob, dataSegment: ND
override def execute(): Unit = {
logInfo(s"Build SEGMENT $segmentId")
- val spanTree = new PartitionSpanningTree(config,
- new PartitionTreeBuilder(dataSegment, readOnlyLayouts, jobId, partitions,
- jobContext.getReadOnlyBuckets.stream.filter(_.getSegmentId.equals(segmentId)).collect(Collectors.toSet[JobBucket])))
- buildParam.setPartitionSpanningTree(spanTree)
-
- val tableDesc = if (jobContext.isPartialBuild) {
- val parser = new IndexDependencyParser(dataModel)
- val relatedTableAlias =
- parser.getRelatedTablesAlias(readOnlyLayouts)
- new PartitionFlatTableDesc(config, dataSegment, spanTree, relatedTableAlias, jobId, partitions)
- } else {
- new PartitionFlatTableDesc(config, dataSegment, spanTree, jobId, partitions)
- }
- buildParam.setTableDesc(tableDesc)
-
- val factTableDS: Dataset[Row] = newFactTableDS()
- buildParam.setFactTableDS(factTableDS)
-
- val fastFactTableDS: Dataset[Row] = newFastFactTableDS()
- buildParam.setFastFactTableDS(fastFactTableDS)
-
+ materializedFactTableView()
if (buildParam.isSkipMaterializedFactTableView) {
onStageSkipped()
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
index e1335978fc..04234294c9 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
@@ -17,25 +17,23 @@
*/
package org.apache.kylin.engine.spark.smarter
-import java.util
-import java.util.Collections
-
+import com.google.common.collect.{Lists, Maps, Sets}
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang3.StringUtils
-import org.apache.kylin.engine.spark.builder.SegmentFlatTable
import org.apache.kylin.engine.spark.job.NSparkCubingUtil
+import org.apache.kylin.engine.spark.job.stage.build.FlatTableAndDictBase
import org.apache.kylin.metadata.cube.model.LayoutEntity
-import org.apache.kylin.metadata.model.{FunctionDesc, JoinTableDesc, NDataModel, TableRef, TblColRef}
+import org.apache.kylin.metadata.model._
import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.{Dataset, Row, SparderEnv, SparkSession}
+import java.util
+import java.util.Collections
import scala.collection.JavaConverters._
import scala.collection.mutable
-import com.google.common.collect.{Lists, Maps, Sets}
-
class IndexDependencyParser(val model: NDataModel) {
private val ccTableNameAliasMap = Maps.newHashMap[String, util.Set[String]]
@@ -113,7 +111,7 @@ class IndexDependencyParser(val model: NDataModel) {
model.getJoinTables.asScala.map((joinTable: JoinTableDesc) => {
joinTableDFMap.put(joinTable, generateDatasetOnTable(ss, joinTable.getTableRef))
})
- val df = SegmentFlatTable.joinFactTableWithLookupTables(rootDF, joinTableDFMap, model, ss)
+ val df = FlatTableAndDictBase.joinFactTableWithLookupTables(rootDF, joinTableDFMap, model, ss)
val filterCondition = model.getFilterCondition
if (StringUtils.isNotEmpty(filterCondition)) {
val massagedCondition = PushDownUtil.massageExpression(model, model.getProject, filterCondition, null)
@@ -127,7 +125,7 @@ class IndexDependencyParser(val model: NDataModel) {
val structType = SchemaProcessor.buildSchemaWithRawTable(tableCols)
val alias = tableRef.getAlias
val dataset = ss.createDataFrame(Lists.newArrayList[Row], structType).alias(alias)
- SegmentFlatTable.wrapAlias(dataset, alias)
+ FlatTableAndDictBase.wrapAlias(dataset, alias)
}
private def initTableNames(): Unit = {
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDimensionTableStat.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDimensionTableStat.scala
index 0a16a971f0..b4ee2087a4 100644
--- a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDimensionTableStat.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDimensionTableStat.scala
@@ -19,13 +19,14 @@
package org.apache.kylin.engine.spark.builder
import org.apache.kylin.common.KylinConfig
-import org.apache.kylin.engine.spark.job.TableMetaManager
-import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
+import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.engine.spark.job.{SegmentJob, TableMetaManager}
import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
import org.apache.kylin.metadata.cube.model._
import org.apache.kylin.metadata.model.SegmentRange
import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession, SparderBaseFunSuite}
+import org.mockito.Mockito
import scala.collection.JavaConverters._
@@ -54,9 +55,11 @@ class TestDimensionTableStat extends SparderBaseFunSuite with SharedSparkSession
val seg = dfMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
val toBuildTree = new AdaptiveSpanningTree(getTestConfig, new AdaptiveTreeBuilder(seg, seg.getIndexPlan.getAllLayouts))
- val flatTableDesc = new SegmentFlatTableDesc(getTestConfig, seg, toBuildTree)
- val flatTable = new SegmentFlatTable(spark, flatTableDesc)
- flatTable.getFlatTableDS
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ val buildParam = new BuildParam()
+ new TestFlatTable(segmentJob, seg, buildParam).test(getTestConfig, toBuildTree)
+
df.getModel.getJoinTables.asScala.foreach { joinTable =>
val dimCount = TableMetaManager.getTableMeta(joinTable.getTable).get.rowCount.get
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestFlatTable.scala
similarity index 57%
copy from src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
copy to src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestFlatTable.scala
index 632ae0cd1c..61cbaf61b0 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestFlatTable.scala
@@ -16,44 +16,37 @@
* limitations under the License.
*/
-package org.apache.kylin.engine.spark.job.stage.build
+package org.apache.kylin.engine.spark.builder
+import org.apache.kylin.common.KylinConfig
import org.apache.kylin.engine.spark.job.SegmentJob
import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.engine.spark.job.stage.build.FlatTableAndDictBase
import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
-import org.apache.kylin.engine.spark.smarter.IndexDependencyParser
import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
import org.apache.kylin.metadata.cube.model.NDataSegment
+import org.apache.kylin.metadata.model.TableRef
import org.apache.spark.sql.{Dataset, Row}
-class MaterializedFactTableView(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
+class TestFlatTable(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends FlatTableAndDictBase(jobContext, dataSegment, buildParam) {
-
- override def execute(): Unit = {
- logInfo(s"Build SEGMENT $segmentId")
- val spanTree = new AdaptiveSpanningTree(config, new AdaptiveTreeBuilder(dataSegment, readOnlyLayouts))
- buildParam.setSpanningTree(spanTree)
-
- val flatTableDesc: SegmentFlatTableDesc = if (jobContext.isPartialBuild) {
- val parser = new IndexDependencyParser(dataModel)
- val relatedTableAlias =
- parser.getRelatedTablesAlias(readOnlyLayouts)
- new SegmentFlatTableDesc(config, dataSegment, spanningTree, relatedTableAlias)
- } else {
- new SegmentFlatTableDesc(config, dataSegment, spanningTree)
- }
+ def test(kylinConfig: KylinConfig, toBuildTree: AdaptiveSpanningTree): Unit = {
+ buildParam.setSpanningTree(toBuildTree)
+ val flatTableDesc = new SegmentFlatTableDesc(kylinConfig, dataSegment, toBuildTree)
buildParam.setFlatTableDesc(flatTableDesc)
-
val factTableDS: Dataset[Row] = newFactTableDS()
buildParam.setFactTableDS(factTableDS)
-
val fastFactTableDS: Dataset[Row] = newFastFactTableDS()
buildParam.setFastFactTableDS(fastFactTableDS)
- if (buildParam.isSkipMaterializedFactTableView) {
- onStageSkipped()
- }
+ val dict: Dataset[Row] = buildDictIfNeed()
+ buildParam.setDict(dict)
+ val flatTable: Dataset[Row] = generateFlatTable()
+ buildParam.setFlatTable(flatTable)
+ val flatTablePart: Dataset[Row] = generateFlatTablePart()
+ buildParam.setFlatTablePart(flatTablePart)
}
- override def getStageName: String = "MaterializedFactTableView"
+ def testNewTableDS(ref: TableRef): Dataset[Row] = {
+ newTableDS(ref)
+ }
}
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestInferFilters.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestInferFilters.scala
index 561521127d..100246e112 100644
--- a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestInferFilters.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestInferFilters.scala
@@ -19,8 +19,9 @@
package org.apache.kylin.engine.spark.builder
import org.apache.kylin.common.KylinConfig
-import org.apache.kylin.engine.spark.job.FiltersUtil
-import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
+import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.engine.spark.job.stage.build.FlatTableAndDictBase
+import org.apache.kylin.engine.spark.job.{FiltersUtil, SegmentJob}
import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
import org.apache.kylin.metadata.cube.model._
@@ -30,6 +31,7 @@ import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession, SparderBa
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
import org.apache.spark.sql.execution.{FilterExec, SparkPlan}
import org.junit.Assert
+import org.mockito.Mockito
import scala.collection.JavaConverters._
import scala.collection.mutable.Set
@@ -46,11 +48,11 @@ class TestInferFilters extends SparderBaseFunSuite with AdaptiveSparkPlanHelper
}
override def beforeEach(): Unit = {
- SegmentFlatTable.inferFiltersEnabled = true
+ FlatTableAndDictBase.inferFiltersEnabled = true
}
override def afterEach(): Unit = {
- SegmentFlatTable.inferFiltersEnabled = false
+ FlatTableAndDictBase.inferFiltersEnabled = false
}
test("infer filters from join desc") {
@@ -64,10 +66,12 @@ class TestInferFilters extends SparderBaseFunSuite with AdaptiveSparkPlanHelper
val seg = dsMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
val toBuildTree = new AdaptiveSpanningTree(getTestConfig, new AdaptiveTreeBuilder(seg, seg.getIndexPlan.getAllLayouts))
- val flatTableDesc = new SegmentFlatTableDesc(getTestConfig, seg, toBuildTree)
- val flatTable = new SegmentFlatTable(spark, flatTableDesc)
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ val buildParam = new BuildParam()
+ new TestFlatTable(segmentJob, seg, buildParam).test(getTestConfig, toBuildTree)
- val filters = getFilterPlan(flatTable.getFlatTableDS.queryExecution.executedPlan)
+ val filters = getFilterPlan(buildParam.getFlatTable.queryExecution.executedPlan)
Assert.assertTrue(Set("EDW.TEST_CAL_DT.CAL_DT", "DEFAULT.TEST_KYLIN_FACT.CAL_DT",
"DEFAULT.TEST_ORDER.TEST_DATE_ENC").subsetOf(FiltersUtil.getAllEqualColSets))
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestSegmentFlatTable.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestSegmentFlatTable.scala
index fda8cc0248..8cf81315c3 100644
--- a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestSegmentFlatTable.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestSegmentFlatTable.scala
@@ -19,9 +19,8 @@
package org.apache.kylin.engine.spark.builder
import org.apache.kylin.common.KylinConfig
-import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
-import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
+import org.apache.kylin.engine.spark.job.SegmentJob
+import org.apache.kylin.engine.spark.job.stage.BuildParam
import org.apache.kylin.metadata.cube.model._
import org.apache.kylin.metadata.model.{SegmentRange, TableDesc, TableRef}
import org.apache.spark.SparkExecutorInfo
@@ -53,16 +52,17 @@ class TestSegmentFlatTable extends SparderBaseFunSuite with SharedSparkSession w
dfMgr.updateDataflow(update)
val seg = dfMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
- val toBuildTree = new AdaptiveSpanningTree(getTestConfig, new AdaptiveTreeBuilder(seg, seg.getIndexPlan.getAllLayouts))
- val flatTableDesc = new SegmentFlatTableDesc(getTestConfig, seg, toBuildTree)
- val flatTable = new SegmentFlatTable(spark, flatTableDesc)
- assert(flatTable.newTableDS(df.getModel.getAllTables.iterator().next()) != null)
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ val buildParam = new BuildParam()
+ val testFlatTable = new TestFlatTable(segmentJob, seg, buildParam)
+ assert(testFlatTable.testNewTableDS(df.getModel.getAllTables.iterator().next()) != null)
val tableRef: TableRef = df.getModel.getAllTables.iterator().next()
val tableDesc: TableDesc = tableRef.getTableDesc
tableDesc.setRangePartition(true)
val ref = new TableRef(df.getModel, tableDesc.getName, tableDesc, false)
- assert(flatTable.newTableDS(ref) != null)
+ assert(testFlatTable.testNewTableDS(ref) != null)
}
test("testSegmentFlatTableWithChineseAndSpecialChar") {
@@ -78,16 +78,17 @@ class TestSegmentFlatTable extends SparderBaseFunSuite with SharedSparkSession w
dfMgr.updateDataflow(update)
val seg = dfMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
- val toBuildTree = new AdaptiveSpanningTree(getTestConfig, new AdaptiveTreeBuilder(seg, seg.getIndexPlan.getAllLayouts))
- val flatTableDesc = new SegmentFlatTableDesc(getTestConfig, seg, toBuildTree)
- val flatTable = new SegmentFlatTable(spark, flatTableDesc)
- assert(flatTable.newTableDS(df.getModel.getAllTables.iterator().next()) != null)
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ val buildParam = new BuildParam()
+ val testFlatTable = new TestFlatTable(segmentJob, seg, buildParam)
+ assert(testFlatTable.testNewTableDS(df.getModel.getAllTables.iterator().next()) != null)
val tableRef: TableRef = df.getModel.getAllTables.iterator().next()
val tableDesc: TableDesc = tableRef.getTableDesc
tableDesc.setRangePartition(true)
val ref = new TableRef(df.getModel, tableDesc.getName, tableDesc, false)
- assert(flatTable.newTableDS(ref) != null)
+ assert(testFlatTable.testNewTableDS(ref) != null)
}
test("waitTillWorkerRegistered") {
@@ -106,13 +107,13 @@ class TestSegmentFlatTable extends SparderBaseFunSuite with SharedSparkSession w
dfMgr.updateDataflow(update)
val seg = dfMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
- val toBuildTree = new AdaptiveSpanningTree(getTestConfig, new AdaptiveTreeBuilder(seg, seg.getIndexPlan.getAllLayouts))
- val flatTableDesc = new SegmentFlatTableDesc(getTestConfig, seg, toBuildTree)
-
val spiedSparkSession = Mockito.spy(spark)
val spiedSparkContext = Mockito.spy(spark.sparkContext)
val spiedTracker = Mockito.spy(spark.sparkContext.statusTracker)
- val flatTable = new SegmentFlatTable(spiedSparkSession, flatTableDesc)
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ val buildParam = new BuildParam()
+ val flatTable = new TestFlatTable(segmentJob, seg, buildParam)
Mockito.when(spiedSparkSession.sparkContext).thenReturn(spiedSparkContext)
Mockito.when(spiedSparkContext.statusTracker).thenReturn(spiedTracker)
Mockito.when(spiedTracker.getExecutorInfos)
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/TestRDSegmentBuildExec.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/TestRDSegmentBuildExec.scala
new file mode 100644
index 0000000000..66dcfdf43b
--- /dev/null
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/TestRDSegmentBuildExec.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.spark.job
+
+import com.google.common.collect.Lists
+import org.apache.kylin.common.util.TestUtils.getTestConfig
+import org.apache.kylin.common.{KapConfig, KylinConfig}
+import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.metadata.cube.model._
+import org.apache.kylin.metadata.model.{NTableMetadataManager, SegmentRange, TableDesc, TableExtDesc}
+import org.apache.spark.sql.SaveMode
+import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession, SparderBaseFunSuite}
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.mockito.Mockito
+import org.scalatest.PrivateMethodTester
+
+import java.util
+import scala.collection.JavaConverters._
+
+class TestRDSegmentBuildExec extends SparderBaseFunSuite with PrivateMethodTester
+ with AdaptiveSparkPlanHelper with SharedSparkSession with LocalMetadata {
+ private val PROJECT = "infer_filter"
+ private val MODEL_NAME1 = "89af4ee2-2cdb-4b07-b39e-4c29856309ab"
+
+ test("test evaluateColumnTotalFromTableDesc") {
+ val dsMgr: NDataflowManager = NDataflowManager.getInstance(getTestConfig, PROJECT)
+ val df: NDataflow = dsMgr.getDataflow(MODEL_NAME1)
+ // cleanup all segments first
+ val update = new NDataflowUpdate(df.getUuid)
+ update.setToRemoveSegsWithArray(df.getSegments.asScala.toArray)
+ dsMgr.updateDataflow(update)
+
+ val seg = dsMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getConfig).thenReturn(getTestConfig)
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ val buildParam = new BuildParam()
+ val exec = new RDSegmentBuildExec(segmentJob, seg, buildParam)
+ exec.initFlatTableOnDetectResource()
+
+ val tableMetadataManager = NTableMetadataManager.getInstance(getTestConfig, PROJECT)
+ var result = exec.evaluateColumnTotalFromTableDesc(tableMetadataManager, 0, "", "UUID")
+ assertEquals(0, result)
+
+ val flatTableDesc = buildParam.getFlatTableDesc
+ val columns = flatTableDesc.getColumns.asScala.toArray
+ var totalRows = 0
+ val columnsIdentity = columns.map(col => col.getIdentity)
+ columns.map(colRef => tableMetadataManager.getTableDesc(flatTableDesc.getTableName(colRef.toString)))
+ .filter(tableMetadataDesc => tableMetadataDesc != null).distinct.foreach(tableMetadataDesc => {
+ val tableName = tableMetadataDesc.getName
+ mockTableStats(tableMetadataDesc, totalRows)
+ if (totalRows % 2 != 0) {
+ val baseDir = KapConfig.getInstanceFromEnv.getMetadataWorkingDirectory
+ val tmp = spark.range(totalRows)
+ tmp.write.mode(SaveMode.Overwrite).parquet(baseDir + tableName + ".parquet")
+ tableMetadataDesc.setLastSnapshotPath(tableName + ".parquet")
+ tableMetadataDesc.setLastSnapshotSize(totalRows)
+ }
+ val colRef = tableMetadataDesc.getColumns.find(columnDesc => columnsIdentity.contains(columnDesc.getIdentity)).get
+ val colIdentity = tableMetadataDesc.getIdentity + "." + colRef.getName
+ result = exec.evaluateColumnTotalFromTableDesc(tableMetadataManager, 0, tableMetadataDesc.getName, colIdentity)
+ assertEquals(totalRows, result)
+ totalRows += 1
+ })
+ }
+
+ private def mockTableStats(tableDesc: TableDesc, totalRows: Int): TableExtDesc = {
+ val tableMetadataManager: NTableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv, PROJECT)
+ var tableExt: TableExtDesc = tableMetadataManager.getOrCreateTableExt(tableDesc)
+ tableExt = tableMetadataManager.copyForWrite(tableExt)
+ val columnStats: util.List[TableExtDesc.ColumnStats] = Lists.newArrayList[TableExtDesc.ColumnStats]
+ for (columnDesc <- tableDesc.getColumns) {
+ if (!columnDesc.isComputedColumn) {
+ var colStats: TableExtDesc.ColumnStats = tableExt.getColumnStatsByName(columnDesc.getName)
+ if (colStats == null) {
+ colStats = new TableExtDesc.ColumnStats
+ colStats.setColumnName(columnDesc.getName)
+ }
+ if ("CAL_DT" == columnDesc.getName) colStats.setCardinality(1000)
+ else if ("TRANS_ID" == columnDesc.getName) colStats.setCardinality(10000)
+ else colStats.setCardinality(100)
+ columnStats.add(colStats)
+ }
+ }
+ if (totalRows % 2 == 0) {
+ tableExt.setTotalRows(totalRows)
+ }
+
+ tableExt.setColumnStats(columnStats)
+ tableMetadataManager.saveTableExt(tableExt)
+ tableExt
+ }
+}
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBaseTest.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBaseTest.scala
new file mode 100644
index 0000000000..1356d19a22
--- /dev/null
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionFlatTableAndDictBaseTest.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.spark.job.stage.build.partition
+
+import org.apache.kylin.common.util.RandomUtil
+import org.apache.kylin.common.util.TestUtils.getTestConfig
+import org.apache.kylin.engine.spark.job.SegmentJob
+import org.apache.kylin.engine.spark.job.stage.BuildParam
+import org.apache.kylin.metadata.cube.model.{NDataflow, NDataflowManager, NDataflowUpdate}
+import org.apache.kylin.metadata.model.SegmentRange
+import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession, SparderBaseFunSuite}
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.junit.jupiter.api.Assertions.{assertNotNull, assertNull}
+import org.mockito.Mockito
+import org.scalatest.PrivateMethodTester
+
+import scala.collection.JavaConverters._
+
+class PartitionFlatTableAndDictBaseTest extends SparderBaseFunSuite with PrivateMethodTester
+ with AdaptiveSparkPlanHelper with SharedSparkSession with LocalMetadata {
+ val modelId = "b780e4e4-69af-449e-b09f-05c90dfa04b6"
+ val project = "default"
+
+ test("test materializedFactTableView") {
+ val dsMgr: NDataflowManager = NDataflowManager.getInstance(getTestConfig, project)
+ val df: NDataflow = dsMgr.getDataflow(modelId)
+ val update = new NDataflowUpdate(df.getUuid)
+ update.setToRemoveSegsWithArray(df.getSegments.asScala.toArray)
+ dsMgr.updateDataflow(update)
+
+ val seg = dsMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getConfig).thenReturn(getTestConfig)
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ Mockito.when(segmentJob.getJobId).thenReturn(RandomUtil.randomUUIDStr())
+ Mockito.when(segmentJob.isPartialBuild).thenReturn(true)
+ val buildParam = new BuildParam()
+ val exec = new PartitionMaterializedFactTableView(segmentJob, seg, buildParam)
+ exec.materializedFactTableView()
+ assertNull(buildParam.getFlatTableDesc)
+ assertNotNull(buildParam.getTableDesc)
+ assertNull(buildParam.getSpanningTree)
+ assertNotNull(buildParam.getPartitionSpanningTree)
+ assertNotNull(buildParam.getFactTableDS)
+ assertNotNull(buildParam.getFastFactTableDS)
+ }
+
+ test("test materializedFactTableView with isPartialBuild{false}") {
+ val dsMgr: NDataflowManager = NDataflowManager.getInstance(getTestConfig, project)
+ val df: NDataflow = dsMgr.getDataflow(modelId)
+ val update = new NDataflowUpdate(df.getUuid)
+ update.setToRemoveSegsWithArray(df.getSegments.asScala.toArray)
+ dsMgr.updateDataflow(update)
+
+ val seg = dsMgr.appendSegment(df, new SegmentRange.TimePartitionedSegmentRange(0L, 1356019200000L))
+ val segmentJob = Mockito.mock(classOf[SegmentJob])
+ Mockito.when(segmentJob.getConfig).thenReturn(getTestConfig)
+ Mockito.when(segmentJob.getSparkSession).thenReturn(spark)
+ Mockito.when(segmentJob.getJobId).thenReturn(RandomUtil.randomUUIDStr())
+ Mockito.when(segmentJob.isPartialBuild).thenReturn(false)
+ val buildParam = new BuildParam()
+ val exec = new PartitionMaterializedFactTableView(segmentJob, seg, buildParam)
+
+ exec.materializedFactTableView()
+ assertNull(buildParam.getFlatTableDesc)
+ assertNotNull(buildParam.getTableDesc)
+ assertNull(buildParam.getSpanningTree)
+ assertNotNull(buildParam.getPartitionSpanningTree)
+ assertNotNull(buildParam.getFactTableDS)
+ assertNotNull(buildParam.getFastFactTableDS)
+ }
+}
[kylin] 22/38: KYLIN-5534 Fix the inconsistent behavior of exporting tds between normal users and admin
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit df6ef0a5aee7c6257835cf64f99a06f4b2f6c34c
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Sat Feb 25 00:07:05 2023 +0800
KYLIN-5534 Fix the inconsistent behavior of exporting tds between normal users and admin
Co-authored-by: sibing.zhang <si...@qq.com>
---
.../rest/controller/open/OpenModelController.java | 5 +-
.../kylin/rest/controller/NModelController.java | 2 +-
.../rest/controller/NModelControllerTest.java | 7 ++-
.../apache/kylin/rest/service/ModelTdsService.java | 12 ++--
.../org/apache/kylin/tool/bisync/BISyncTool.java | 4 +-
.../apache/kylin/tool/bisync/SyncModelBuilder.java | 5 --
.../service/ModelTdsServiceColumnNameTest.java | 3 +-
.../kylin/rest/service/ModelTdsServiceTest.java | 68 +++++++++++++++++++---
.../kylin/tool/bisync/SyncModelBuilderTest.java | 2 +-
.../tool/bisync/tableau/TableauDatasourceTest.java | 5 +-
10 files changed, 83 insertions(+), 30 deletions(-)
diff --git a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
index 3b10bfc29a..b28319abeb 100644
--- a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
+++ b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
@@ -73,7 +73,6 @@ import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ModelTdsService;
-import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.tool.bisync.SyncContext;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.util.DataRangeUtils;
@@ -414,9 +413,7 @@ public class OpenModelController extends NBasicController {
}
SyncContext syncContext = tdsService.prepareSyncContext(projectName, modelId, exportAs, element, host, port);
- SyncModel syncModel = AclPermissionUtil.isAdmin()
- ? tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measures)
- : tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures);
+ SyncModel syncModel = tdsService.exportModel(syncContext);
tdsService.preCheckNameConflict(syncModel);
tdsService.dumpSyncModel(syncContext, syncModel, response);
}
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
index e5029e98f9..f05dd6b9f9 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
@@ -34,8 +34,8 @@ import java.util.Locale;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.PartitionDesc;
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
index ef405752e1..ffe814c4d7 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
@@ -84,6 +84,7 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -783,7 +784,8 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
SyncModel syncModel = Mockito.mock(SyncModel.class);
Mockito.doReturn(syncContext).when(tdsService).prepareSyncContext(project, modelName,
SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Mockito.doReturn(syncModel).when(tdsService).exportModel(syncContext);
+ Mockito.doReturn(syncModel).when(tdsService).exportTDSDimensionsAndMeasuresByAdmin(syncContext,
+ ImmutableList.of(), ImmutableList.of());
Mockito.doReturn(Boolean.TRUE).when(tdsService).preCheckNameConflict(syncModel);
mockMvc.perform(MockMvcRequestBuilders.get("/api/models/validate_export").param("model", modelName)
.param("project", project).contentType(MediaType.APPLICATION_JSON))
@@ -805,7 +807,8 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
syncContext.setKylinConfig(getTestConfig());
syncContext.setAdmin(true);
SyncModel syncModel = Mockito.mock(SyncModel.class);
- Mockito.doReturn(syncModel).when(tdsService).exportModel(syncContext);
+ Mockito.doReturn(syncModel).when(tdsService).exportTDSDimensionsAndMeasuresByAdmin(syncContext,
+ ImmutableList.of(), ImmutableList.of());
mockMvc.perform(MockMvcRequestBuilders.get("/api/models/{model}/export", modelName).param("project", project)
.param("export_as", "TABLEAU_CONNECTOR_TDS").param("element", "AGG_INDEX_AND_TABLE_INDEX_COL")
.param("server_host", "localhost").param("server_port", "8080").contentType(MediaType.APPLICATION_JSON)
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
index b28fe7a019..84e611fb96 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
@@ -69,6 +69,7 @@ import org.apache.kylin.tool.bisync.model.MeasureDef;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.springframework.stereotype.Component;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
@@ -111,7 +112,6 @@ public class ModelTdsService extends AbstractModelService {
Set<String> measureNames = syncModel.getMetrics().stream().filter(measureDef -> !measureDef.isHidden())
.map(measureDef -> measureDef.getMeasure().getName()).collect(Collectors.toSet());
Map<String, ColumnDef> nameOfColDefMap = syncModel.getColumnDefMap().values().stream()
- .filter(columnDef -> !columnDef.isHidden())
.collect(Collectors.toMap(ColumnDef::getAliasDotColumn, Function.identity()));
nameOfColDefMap.forEach((aliasColName, columnDef) -> {
@@ -129,9 +129,9 @@ public class ModelTdsService extends AbstractModelService {
}
public SyncModel exportModel(SyncContext syncContext) {
- checkModelExportPermission(syncContext.getProjectName(), syncContext.getModelId());
- checkModelPermission(syncContext.getProjectName(), syncContext.getModelId());
- return new SyncModelBuilder(syncContext).buildSourceSyncModel();
+ return AclPermissionUtil.isAdmin()
+ ? exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of())
+ : exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(), ImmutableList.of());
}
public SyncModel exportTDSDimensionsAndMeasuresByNormalUser(SyncContext syncContext, List<String> dimensions,
@@ -162,8 +162,8 @@ public class ModelTdsService extends AbstractModelService {
checkTableHasColumnPermission(syncContext.getModelElement(), project, modelId, authorizedCols, dimensions,
measures);
- return new SyncModelBuilder(syncContext).buildHasPermissionSourceSyncModel(authTables, authColumns, dimensions,
- measures);
+ return new SyncModelBuilder(syncContext).buildHasPermissionSourceSyncModel(authTables, authorizedCols,
+ dimensions, measures);
}
public SyncModel exportTDSDimensionsAndMeasuresByAdmin(SyncContext syncContext, List<String> dimensions,
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
index 4834af6595..039ff1dff7 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
@@ -25,6 +25,7 @@ import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.tool.bisync.tableau.TableauDataSourceConverter;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
public class BISyncTool {
@@ -33,7 +34,8 @@ public class BISyncTool {
@VisibleForTesting
public static BISyncModel dumpToBISyncModel(SyncContext syncContext) {
- SyncModel syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel();
+ SyncModel syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel(ImmutableList.of(),
+ ImmutableList.of());
return getBISyncModel(syncContext, syncModel);
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
index fa14d623cd..de00e0ffaf 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
@@ -44,7 +44,6 @@ import org.apache.kylin.tool.bisync.model.JoinTreeNode;
import org.apache.kylin.tool.bisync.model.MeasureDef;
import org.apache.kylin.tool.bisync.model.SyncModel;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -57,10 +56,6 @@ public class SyncModelBuilder {
this.syncContext = syncContext;
}
- public SyncModel buildSourceSyncModel() {
- return buildSourceSyncModel(ImmutableList.of(), ImmutableList.of());
- }
-
public SyncModel buildSourceSyncModel(List<String> dimensions, List<String> measures) {
NDataModel dataModelDesc = syncContext.getDataflow().getModel();
IndexPlan indexPlan = syncContext.getDataflow().getIndexPlan();
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
index db24394952..1047372feb 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.service;
import lombok.extern.slf4j.Slf4j;
+import com.google.common.collect.ImmutableList;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.engine.spark.utils.SparkJobFactoryUtils;
import org.apache.kylin.junit.rule.TransactionExceptedException;
@@ -122,7 +123,7 @@ public class ModelTdsServiceColumnNameTest extends SourceTestCase {
syncContext.setAdmin(true);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), getProject()).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
- SyncModel syncModel = tdsService.exportModel(syncContext);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
}
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
index b60753ea7b..885408a725 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
@@ -43,6 +43,7 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.rest.constant.Constant;
@@ -160,10 +161,21 @@ public class ModelTdsServiceTest extends SourceTestCase {
syncContext.setAdmin(true);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
- SyncModel syncModel = tdsService.exportModel(syncContext);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
+ ImmutableList.of());
Assert.assertThrows(
"There are duplicated names among dimension column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
KylinException.class, () -> tdsService.preCheckNameConflict(syncModel));
+
+ syncContext.setAdmin(false);
+ prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ SyncModel syncModel2 = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(),
+ ImmutableList.of());
+ Assert.assertThrows(
+ "There are duplicated names among dimension column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
+ KylinException.class, () -> tdsService.preCheckNameConflict(syncModel2));
}
@Test
@@ -187,7 +199,16 @@ public class ModelTdsServiceTest extends SourceTestCase {
syncContext.setAdmin(true);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
- SyncModel syncModel = tdsService.exportModel(syncContext);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
+ ImmutableList.of());
+ Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
+
+ syncContext.setAdmin(false);
+ prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ syncModel = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(),
+ ImmutableList.of());
Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
}
@@ -212,8 +233,21 @@ public class ModelTdsServiceTest extends SourceTestCase {
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
syncContext.setAdmin(true);
- SyncModel syncModel = tdsService.exportModel(syncContext);
- Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
+ ImmutableList.of());
+ Assert.assertThrows(
+ "There are duplicated names among model column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
+ KylinException.class, () -> tdsService.preCheckNameConflict(syncModel));
+
+ syncContext.setAdmin(false);
+ prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ SyncModel syncModel2 = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(),
+ ImmutableList.of());
+ Assert.assertThrows(
+ "There are duplicated names among model column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
+ KylinException.class, () -> tdsService.preCheckNameConflict(syncModel2));
}
@Test
@@ -310,7 +344,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
thrown.expectMessage("current user does not have full permission on requesting model");
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- tdsService.exportModel(syncContext);
+ tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
} finally {
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
@@ -359,7 +393,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost",
8080);
- tdsService.exportModel(syncContext);
+ tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
} finally {
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
@@ -449,7 +483,8 @@ public class ModelTdsServiceTest extends SourceTestCase {
prepareBasic(project);
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL, "localhost", 7070);
- SyncModel syncModel = tdsService.exportModel(syncContext);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
+ ImmutableList.of());
TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext, syncModel);
ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
datasource1.dump(outStream4);
@@ -492,6 +527,25 @@ public class ModelTdsServiceTest extends SourceTestCase {
manager.updateAclTCR(g1a1, "g1", false);
}
+ private void prepareBasicPermissionByModel(String project, NDataModel model) {
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+ AclTCR u1a1 = new AclTCR();
+ AclTCR.Table u1t1 = new AclTCR.Table();
+ for (TableRef table : model.getAllTables()) {
+ AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c1 = new AclTCR.Column();
+ List<String> colNames = Lists.newArrayList();
+ for (TblColRef col : table.getColumns()) {
+ colNames.add(col.getName());
+ }
+ u1c1.addAll(colNames);
+ u1cr1.setColumn(u1c1);
+ u1t1.put(table.getTableIdentity(), u1cr1);
+ }
+ u1a1.setTable(u1t1);
+ manager.updateAclTCR(u1a1, "u1", true);
+ }
+
@Test
public void testCheckTablePermission() {
val project = "default";
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
index d45eb7b97c..6844db70ae 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
@@ -76,7 +76,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
val syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
syncContext.setAdmin(true);
- val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel();
+ val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel(ImmutableList.of(), ImmutableList.of());
val df = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project).getDataflow(modelId);
val model = df.getModel();
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java
index 43839bcd6d..53bb8b8cd7 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java
@@ -36,6 +36,7 @@ import org.junit.Before;
import org.junit.Test;
import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
import com.google.common.io.CharStreams;
import lombok.val;
@@ -72,7 +73,7 @@ public class TableauDatasourceTest extends NLocalFileMetadataTestCase {
val project = "default";
val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
val syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
- val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel();
+ val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel(ImmutableList.of(), ImmutableList.of());
TableauDatasourceModel datasource = new TableauDataSourceConverter().convert(syncModel, syncContext);
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
@@ -86,7 +87,7 @@ public class TableauDatasourceTest extends NLocalFileMetadataTestCase {
val project = "default";
val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
val syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
- val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel();
+ val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel(ImmutableList.of(), ImmutableList.of());
TableauDatasourceModel datasource = new TableauDataSourceConverter().convert(syncModel, syncContext);
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
[kylin] 29/38: KYLIN-5534 [FOLLOW UP] Set 'kylin.model.skip-check-tds' to true
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9af54c048448711da88658430af2bd147bc8989b
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Thu Mar 2 14:42:02 2023 +0800
KYLIN-5534 [FOLLOW UP] Set 'kylin.model.skip-check-tds' to true
Co-authored-by: sibing.zhang <si...@qq.com>
---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../rest/controller/open/OpenModelController.java | 5 ++-
.../apache/kylin/rest/service/ModelTdsService.java | 16 ++++----
.../service/ModelTdsServiceColumnNameTest.java | 3 +-
.../kylin/rest/service/ModelTdsServiceTest.java | 45 ++++++++++++++--------
5 files changed, 45 insertions(+), 26 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 19021c4d73..fccc636084 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3730,7 +3730,7 @@ public abstract class KylinConfigBase implements Serializable {
}
public boolean skipCheckTds() {
- return Boolean.parseBoolean(getOptional("kylin.model.skip-check-tds", FALSE));
+ return Boolean.parseBoolean(getOptional("kylin.model.skip-check-tds", TRUE));
}
public boolean isHdfsMetricsPeriodicCalculationEnabled() {
diff --git a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
index b28319abeb..3b10bfc29a 100644
--- a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
+++ b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
@@ -73,6 +73,7 @@ import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ModelTdsService;
+import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.tool.bisync.SyncContext;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.util.DataRangeUtils;
@@ -413,7 +414,9 @@ public class OpenModelController extends NBasicController {
}
SyncContext syncContext = tdsService.prepareSyncContext(projectName, modelId, exportAs, element, host, port);
- SyncModel syncModel = tdsService.exportModel(syncContext);
+ SyncModel syncModel = AclPermissionUtil.isAdmin()
+ ? tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measures)
+ : tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures);
tdsService.preCheckNameConflict(syncModel);
tdsService.dumpSyncModel(syncContext, syncModel, response);
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
index 84e611fb96..847d22905c 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
@@ -129,9 +129,15 @@ public class ModelTdsService extends AbstractModelService {
}
public SyncModel exportModel(SyncContext syncContext) {
- return AclPermissionUtil.isAdmin()
- ? exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of())
- : exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(), ImmutableList.of());
+ if (AclPermissionUtil.isAdmin()) {
+ return exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
+ }
+
+ String projectName = syncContext.getProjectName();
+ String modelId = syncContext.getModelId();
+ checkModelExportPermission(projectName, modelId);
+ checkModelPermission(projectName, modelId);
+ return exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(), ImmutableList.of());
}
public SyncModel exportTDSDimensionsAndMeasuresByNormalUser(SyncContext syncContext, List<String> dimensions,
@@ -168,10 +174,6 @@ public class ModelTdsService extends AbstractModelService {
public SyncModel exportTDSDimensionsAndMeasuresByAdmin(SyncContext syncContext, List<String> dimensions,
List<String> measures) {
- String projectName = syncContext.getProjectName();
- String modelId = syncContext.getModelId();
- checkModelExportPermission(projectName, modelId);
- checkModelPermission(projectName, modelId);
return new SyncModelBuilder(syncContext).buildSourceSyncModel(dimensions, measures);
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
index 1047372feb..6131b84e70 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
@@ -123,7 +123,8 @@ public class ModelTdsServiceColumnNameTest extends SourceTestCase {
syncContext.setAdmin(true);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), getProject()).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
- SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ overwriteSystemProp("kylin.model.skip-check-tds", "false");
Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
}
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
index 885408a725..03f6fa39c3 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
@@ -161,21 +161,34 @@ public class ModelTdsServiceTest extends SourceTestCase {
syncContext.setAdmin(true);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
- SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ overwriteSystemProp("kylin.model.skip-check-tds", "false");
Assert.assertThrows(
"There are duplicated names among dimension column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
KylinException.class, () -> tdsService.preCheckNameConflict(syncModel));
syncContext.setAdmin(false);
prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
+ .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncModel syncModel2 = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ SyncModel syncModel2 = tdsService.exportModel(syncContext);
Assert.assertThrows(
"There are duplicated names among dimension column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
KylinException.class, () -> tdsService.preCheckNameConflict(syncModel2));
+
+ overwriteSystemProp("kylin.model.skip-check-tds", "true");
+ Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
+
+ syncContext.setAdmin(false);
+ prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
+ .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ SyncModel syncModel3 = tdsService.exportModel(syncContext);
+ Assert.assertTrue(tdsService.preCheckNameConflict(syncModel3));
}
@Test
@@ -199,16 +212,16 @@ public class ModelTdsServiceTest extends SourceTestCase {
syncContext.setAdmin(true);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
- SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ SyncModel syncModel = tdsService.exportModel(syncContext);
Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
syncContext.setAdmin(false);
prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
+ .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- syncModel = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ syncModel = tdsService.exportModel(syncContext);
Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
}
@@ -233,18 +246,19 @@ public class ModelTdsServiceTest extends SourceTestCase {
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
syncContext.setKylinConfig(getTestConfig());
syncContext.setAdmin(true);
- SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ overwriteSystemProp("kylin.model.skip-check-tds", "false");
+ SyncModel syncModel = tdsService.exportModel(syncContext);
Assert.assertThrows(
"There are duplicated names among model column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
KylinException.class, () -> tdsService.preCheckNameConflict(syncModel));
syncContext.setAdmin(false);
prepareBasicPermissionByModel(projectName, syncContext.getDataflow().getModel());
+ Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
+ .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncModel syncModel2 = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ SyncModel syncModel2 = tdsService.exportModel(syncContext);
Assert.assertThrows(
"There are duplicated names among model column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
KylinException.class, () -> tdsService.preCheckNameConflict(syncModel2));
@@ -344,7 +358,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
thrown.expectMessage("current user does not have full permission on requesting model");
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
+ tdsService.exportModel(syncContext);
} finally {
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
@@ -393,7 +407,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost",
8080);
- tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(), ImmutableList.of());
+ tdsService.exportModel(syncContext);
} finally {
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
@@ -483,8 +497,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
prepareBasic(project);
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL, "localhost", 7070);
- SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, ImmutableList.of(),
- ImmutableList.of());
+ SyncModel syncModel = tdsService.exportModel(syncContext);
TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext, syncModel);
ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
datasource1.dump(outStream4);
[kylin] 08/38: KYLIN-5525 fix error when model init error in multi threads
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 09ea4572781fcc0ea9da76204e3fdce52d8f85c6
Author: ChenLiang.Lu <31...@users.noreply.github.com>
AuthorDate: Thu Feb 16 10:09:42 2023 +0800
KYLIN-5525 fix error when model init error in multi threads
---
.../kylin/metadata/model/ColExcludedChecker.java | 4 +++-
.../apache/kylin/metadata/model/NDataModel.java | 4 ++++
.../realization/RealizationRuntimeException.java | 28 ++++++++++++++++++++++
.../org/apache/kylin/newten/TableIndexTest.java | 19 +++++++++++++++
.../kylin/query/routing/RealizationChooser.java | 3 +++
5 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
index 8fd68f26af..97b3c3e8c6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
@@ -88,7 +88,9 @@ public class ColExcludedChecker {
if (model == null || model.isBroken()) {
return;
}
- model.init(config, project, Lists.newArrayList());
+ if (!model.isInitAlready()) {
+ model.init(config, project, Lists.newArrayList());
+ }
model.getAllTables().stream().filter(Objects::nonNull) //
.flatMap(tableRef -> tableRef.getColumns().stream())
.filter(tblColRef -> excludedCols.contains(tblColRef.getColumnDesc()))
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index c51b0cd5ff..fbb704a871 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -280,6 +280,9 @@ public class NDataModel extends RootPersistentEntity {
// mark this model as used for model save checking
private boolean saveCheck = false;
+ // mark this model has invoked init() function
+ private boolean initAlready = false;
+
public enum ColumnStatus {
TOMB, EXIST, DIMENSION
}
@@ -1026,6 +1029,7 @@ public class NDataModel extends RootPersistentEntity {
throw new KylinException(TABLE_JOIN_RELATIONSHIP_ERROR,
MsgPicker.getMsg().getDimensionTableUsedInThisModel());
}
+ this.setInitAlready(true);
}
public ComputedColumnUtil.CCConflictInfo checkCCFailAtEnd(KylinConfig config, String project,
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java
new file mode 100644
index 0000000000..aff464f813
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.realization;
+
+public class RealizationRuntimeException extends RuntimeException {
+
+ private static final long serialVersionUID = 7631508437415520091L;
+
+ public RealizationRuntimeException(String message, Throwable t) {
+ super(message, t);
+ }
+}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java
index 3c94b1f00c..1d9200f8d9 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java
@@ -29,9 +29,12 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
+import org.apache.kylin.metadata.realization.RealizationRuntimeException;
import org.apache.kylin.util.ExecAndComp;
import org.apache.kylin.util.ExecAndComp.CompareLevel;
+
import org.apache.spark.sql.SparderEnv;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -172,4 +175,20 @@ public class TableIndexTest extends NLocalWithSparkSessionTest {
+ "WHERE KYLIN_FACT.cal_dt = DATE '2012-01-01' "));
ExecAndComp.execAndCompare(query, getProject(), CompareLevel.SAME, "left");
}
+
+ @Test
+ public void testUseTableIndexUnionQuery() throws Exception {
+ overwriteSystemProp("kylin.query.use-tableindex-answer-non-raw-query", "true");
+ overwriteSystemProp("kylin.metadata.table-exclusion-enabled", "true");
+ fullBuild("acfde546-2cc9-4eec-bc92-e3bd46d4e2ee");
+ populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
+ List<Pair<String, String>> query = new ArrayList<>();
+
+ query.add(Pair.newPair("query_table_index2", "select sum(PRICE) from TEST_KYLIN_FACT group by PRICE"
+ + " union all select sum(PRICE) from TEST_KYLIN_FACT group by PRICE"));
+ ExecAndComp.execAndCompare(query, getProject(), CompareLevel.SAME, "left");
+ RealizationRuntimeException error = new RealizationRuntimeException("unexpected error", new RuntimeException(
+ "error"));
+ assert error.getMessage().contains("unexpected error");
+ }
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index d4f70edd52..389eb8b074 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -95,6 +95,7 @@ import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
+import org.apache.kylin.metadata.realization.RealizationRuntimeException;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPContextProp;
@@ -185,6 +186,8 @@ public class RealizationChooser {
throw (NoRealizationFoundException) e.getCause();
} else if (e.getCause() instanceof NoStreamingRealizationFoundException) {
throw (NoStreamingRealizationFoundException) e.getCause();
+ } else {
+ throw new RealizationRuntimeException("unexpected error when choose layout", e);
}
} catch (InterruptedException e) {
for (Future<?> future : futureList) {
[kylin] 20/38: KYLIN-5532 When calcite optimize, can cancel query search
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 96d806eae898e912494df714b03f3b12ce2a21ba
Author: jlf <lo...@kyligence.io>
AuthorDate: Fri Feb 24 22:27:19 2023 +0800
KYLIN-5532 When calcite optimize, can cancel query search
---
.../org/apache/kylin/query/SlowQueryDetector.java | 9 ++++++--
.../apache/kylin/rest/service/QueryService.java | 1 +
.../kylin/rest/service/QueryServiceTest.java | 24 ++++++++++++++++++++++
3 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java b/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java
index 3810c2d09f..136127a3b3 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java
@@ -21,6 +21,7 @@ package org.apache.kylin.query;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.calcite.util.CancelFlag;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.slf4j.Logger;
@@ -78,8 +79,9 @@ public class SlowQueryDetector extends Thread {
if (QueryContext.current().getQueryTagInfo().isAsyncQuery()) {
return;
}
- runningQueries.put(currentThread(), new QueryEntry(System.currentTimeMillis(), currentThread(),
- QueryContext.current().getQueryId(), QueryContext.current().getUserSQL(), stopId, false));
+ runningQueries.put(currentThread(),
+ new QueryEntry(System.currentTimeMillis(), currentThread(), QueryContext.current().getQueryId(),
+ QueryContext.current().getUserSQL(), stopId, false, CancelFlag.getContextCancelFlag()));
}
public void queryEnd() {
@@ -112,6 +114,7 @@ public class SlowQueryDetector extends Thread {
// interrupt query thread if Stop By User but running
for (QueryEntry e : runningQueries.values()) {
if (e.isStopByUser) {
+ e.getPlannerCancelFlag().requestCancel();
e.getThread().interrupt();
logger.error("Trying to cancel query: {}", e.getThread().getName());
}
@@ -162,6 +165,7 @@ public class SlowQueryDetector extends Thread {
final String sql;
final String stopId;
boolean isStopByUser;
+ final CancelFlag plannerCancelFlag;
public long getRunningTime() {
return (System.currentTimeMillis() - startTime) / 1000;
@@ -170,6 +174,7 @@ public class SlowQueryDetector extends Thread {
private boolean setInterruptIfTimeout() {
long runningMs = System.currentTimeMillis() - startTime;
if (runningMs >= queryTimeoutMs) {
+ plannerCancelFlag.requestCancel();
thread.interrupt();
logger.error("Trying to cancel query: {}", thread.getName());
return true;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 42fa8425b5..9a3c19a10c 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -336,6 +336,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
if (e.getStopId().equals(id)) {
logger.error("Trying to cancel query: {}", e.getThread().getName());
e.setStopByUser(true);
+ e.getPlannerCancelFlag().requestCancel();
e.getThread().interrupt();
break;
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 4dd8eff89f..4f1c9c1fba 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -18,11 +18,13 @@
package org.apache.kylin.rest.service;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.kylin.common.QueryContext.PUSHDOWN_HIVE;
import static org.apache.kylin.common.QueryTrace.EXECUTION;
import static org.apache.kylin.common.QueryTrace.SPARK_JOB_EXECUTION;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_NOT_EXIST;
import static org.apache.kylin.rest.metrics.QueryMetricsContextTest.getInfluxdbFields;
+import static org.awaitility.Awaitility.await;
import static org.springframework.security.acls.domain.BasePermission.ADMINISTRATION;
import java.io.ByteArrayInputStream;
@@ -101,6 +103,7 @@ import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.query.SlowQueryDetector;
import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.query.blacklist.SQLBlacklistItem;
import org.apache.kylin.query.blacklist.SQLBlacklistManager;
@@ -133,6 +136,7 @@ import org.apache.kylin.rest.util.QueryCacheSignatureUtil;
import org.apache.kylin.rest.util.SpringContext;
import org.apache.kylin.source.adhocquery.PushdownResult;
import org.apache.spark.sql.SparkSession;
+import org.awaitility.Duration;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -2698,4 +2702,24 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
Object routeToCalcite2 = isCalciteEngineCapable.invoke(queryExec, rel2);
Assert.assertEquals(true, routeToCalcite2);
}
+
+ @Test
+ public void testStop() {
+ val stopId = RandomUtil.randomUUIDStr();
+ val execute = new Thread(() -> {
+ QueryContext.current().setQueryId(RandomUtil.randomUUIDStr());
+ QueryContext.current().setProject("default");
+ QueryContext.current().setUserSQL("select 1");
+ queryService.slowQueryDetector.queryStart(stopId);
+ await().pollDelay(new Duration(5, SECONDS)).until(() -> true);
+ });
+ execute.start();
+ await().pollDelay(new Duration(1, SECONDS)).until(() -> true);
+ queryService.stopQuery(stopId);
+ val result = SlowQueryDetector.getRunningQueries().values().stream()
+ .filter(entry -> StringUtils.equals(stopId, entry.getStopId())).findFirst();
+ Assert.assertTrue(result.isPresent());
+ val queryEntry = result.get();
+ Assert.assertTrue(queryEntry.getPlannerCancelFlag().isCancelRequested());
+ }
}
[kylin] 16/38: [DIRTY] Fix a build job is submitted in yarn cluster mode, it cannot be written into the driver log
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 80a87f9d531d9325e5fdf1a0af839ba9f601dcef
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Fri Feb 24 13:55:21 2023 +0800
[DIRTY] Fix a build job is submitted in yarn cluster mode, it cannot be written into the driver log
Co-authored-by: sibing.zhang <si...@qq.com>
---
.../java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../org/apache/kylin/common/KylinConfigBaseTest.java | 9 +++++++++
.../apache/kylin/job/execution/NExecutableManager.java | 5 +++++
.../org/apache/kylin/rest/service/JobServiceTest.java | 18 ++++++++++++++++++
4 files changed, 36 insertions(+)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 853c5d6ce6..7f90613fad 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3780,6 +3780,10 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.build.segment-overlap-enabled", FALSE));
}
+ public boolean isJobTmpDirALLPermissionEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.job-tmp-dir-all-permission-enabled", FALSE));
+ }
+
public boolean isProjectMergeWithBloatEnabled() {
return Boolean.parseBoolean(getOptional("kylin.query.project-merge-with-bloat-enabled", "true"));
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 23f39ccd1f..5dfad30ca5 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -1397,6 +1397,15 @@ class KylinConfigBaseTest {
assertTrue(config.isBuildSegmentOverlapEnabled());
}
+ @Test
+ void testIsJobTmpDirReadWritePermissionEnabled() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.engine.job-tmp-dir-all-permission-enabled", "false");
+ assertFalse(config.isJobTmpDirALLPermissionEnabled());
+ config.setProperty("kylin.engine.job-tmp-dir-all-permission-enabled", "true");
+ assertTrue(config.isJobTmpDirALLPermissionEnabled());
+ }
+
@Test
void testIsQuotaStorageEnabled() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
index 4f1b192f1d..31d1b9ead7 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
@@ -62,6 +62,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
@@ -1535,6 +1537,9 @@ public class NExecutableManager {
Path path = new Path(resPath);
FileSystem fs = HadoopUtil.getWorkingFileSystem();
dout = fs.create(path, true);
+ if (KylinConfig.getInstanceFromEnv().isJobTmpDirALLPermissionEnabled()) {
+ fs.setPermission(path.getParent(), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.ALL));
+ }
JsonUtil.writeValue(dout, obj);
} catch (Exception e) {
// the operation to update output to hdfs failed, next task should not be interrupted.
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 1bb130d731..27245e7046 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -61,7 +61,9 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.metadata.Epoch;
@@ -1903,6 +1905,22 @@ public class JobServiceTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId));
}
+ @Test
+ public void testJobSubdirectoryPermission() throws IOException {
+ String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de";
+ NExecutableManager manager = NExecutableManager.getInstance(jobService.getConfig(), "default");
+ ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
+ Map<String, String> info = Maps.newHashMap();
+ info.put("nodes", "localhost:7070:all");
+ executableOutputPO.setInfo(info);
+ overwriteSystemProp("kylin.engine.job-tmp-dir-all-permission-enabled", "true");
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
+ String file = KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId);
+ Path path = new Path(file);
+ manager.updateJobOutputToHDFS(file, executableOutputPO);
+ Assert.assertSame(FsAction.ALL, fs.getFileStatus(path.getParent()).getPermission().getOtherAction());
+ }
+
@Test
public void testExecutableResponse() throws Exception {
val modelManager = mock(NDataModelManager.class);
[kylin] 07/38: KYLIN-5524 Supports CONCAT for variable arguments
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f929bd876ed54afbd1490d2717894288fd197ec6
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Tue Feb 14 13:54:41 2023 +0800
KYLIN-5524 Supports CONCAT for variable arguments
---
.../kylin/query/udf/stringUdf/ConcatUDF.java | 45 +++++++++++++++++++---
.../org/apache/kylin/query/udf/StringUDFTest.java | 21 +++++++---
.../kylin/query/runtime/ExpressionConverter.scala | 2 +-
3 files changed, 57 insertions(+), 11 deletions(-)
diff --git a/src/query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java b/src/query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java
index 01f16d47e6..841cc16bfe 100644
--- a/src/query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java
+++ b/src/query/src/main/java/org/apache/kylin/query/udf/stringUdf/ConcatUDF.java
@@ -18,14 +18,49 @@
package org.apache.kylin.query.udf.stringUdf;
-import org.apache.calcite.linq4j.function.Parameter;
+import org.apache.calcite.adapter.enumerable.CallImplementor;
+import org.apache.calcite.adapter.enumerable.UdfMethodNameImplementor;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.fun.udf.UdfDef;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.sql.type.SqlOperandCountRanges;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeTransforms;
-public class ConcatUDF {
+import java.util.Arrays;
+import java.util.stream.Collectors;
- public String CONCAT(@Parameter(name = "col1") Object col1, @Parameter(name = "col2") Object col2) {
- if (col1 == null || col2 == null) {
+public class ConcatUDF implements UdfDef {
+
+ private static final String FUNC_NAME = "CONCAT";
+
+ private ConcatUDF() {
+ throw new IllegalStateException("Utility class");
+ }
+
+ public static final SqlFunction OPERATOR = new SqlFunction(new SqlIdentifier(FUNC_NAME, SqlParserPos.ZERO),
+ ReturnTypes.cascade(opBinding -> {
+ int precision = opBinding.collectOperandTypes().stream().mapToInt(RelDataType::getPrecision).sum();
+ return opBinding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR, precision);
+ }, SqlTypeTransforms.TO_NULLABLE),
+ (callBinding, returnType, operandTypes) -> Arrays.fill(operandTypes,
+ callBinding.getTypeFactory().createJavaType(Object.class)),
+ OperandTypes.repeat(SqlOperandCountRanges.from(2), OperandTypes.ANY), null,
+ SqlFunctionCategory.USER_DEFINED_FUNCTION);
+
+ public static final CallImplementor IMPLEMENTOR = new UdfMethodNameImplementor(FUNC_NAME.toLowerCase(),
+ ConcatUDF.class);
+
+ public static String concat(Object... args) {
+ if (args == null) {
return null;
}
- return "" + col1 + col2;
+
+ return Arrays.stream(args).map(String::valueOf).collect(Collectors.joining());
}
}
diff --git a/src/query/src/test/java/org/apache/kylin/query/udf/StringUDFTest.java b/src/query/src/test/java/org/apache/kylin/query/udf/StringUDFTest.java
index 543795a2aa..799660a37f 100644
--- a/src/query/src/test/java/org/apache/kylin/query/udf/StringUDFTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/udf/StringUDFTest.java
@@ -33,16 +33,27 @@ import org.junit.Test;
public class StringUDFTest {
@Test
- public void testConcatUDF() throws Exception {
- ConcatUDF cu = new ConcatUDF();
- String str1 = cu.CONCAT("Apache ", "Kylin");
+ public void testConcatUDF() {
+ String str1 = ConcatUDF.concat("Apache ", "Kylin");
assertEquals("Apache Kylin", str1);
- String str2 = cu.CONCAT("", "Kylin");
+ String str2 = ConcatUDF.concat("", "Kylin");
assertEquals("Kylin", str2);
- String str3 = cu.CONCAT("Apache", "");
+ String str3 = ConcatUDF.concat("Apache", "");
assertEquals("Apache", str3);
+
+ String str4 = ConcatUDF.concat("Apache", 1);
+ assertEquals("Apache1", str4);
+
+ String str5 = ConcatUDF.concat(1, 1);
+ assertEquals("11", str5);
+
+ String str6 = ConcatUDF.concat("Apache ", "Kylin", " Kyligence");
+ assertEquals("Apache Kylin Kyligence", str6);
+
+ String str7 = ConcatUDF.concat(null);
+ assertNull(str7);
}
@Test
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
index a839116059..856c19f50f 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
@@ -284,7 +284,7 @@ object ExpressionConverter {
else children.apply(2).asInstanceOf[Int]
new Column(StringLocate(k_lit(children.head).expr, k_lit(children.apply(1)).expr, lit(pos).expr)) //position(substr,str,start)
case "concat" =>
- concat(k_lit(children.head), k_lit(children.apply(1)))
+ concat(children.map(k_lit): _*)
case "concat_ws" =>
concat_ws(children.head.toString, k_lit(children.apply(1)))
case "split_part" =>
[kylin] 33/38: KYLIN-5530 [FOLLOW UP] fix config kylin.engine.persist-flat-use-snapshot-enabled
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b10a87b65d4e2127f2975baacd9792128c8f5fd9
Author: Mingming Ge <7m...@gmail.com>
AuthorDate: Tue Mar 7 11:13:54 2023 +0800
KYLIN-5530 [FOLLOW UP] fix config kylin.engine.persist-flat-use-snapshot-enabled
---
.../kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 93df9cf9a4..eba207c7bc 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -435,7 +435,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
val fs = HadoopUtil.getWorkingFileSystem
if (snapshotResPath == null
|| !fs.exists(snapshotResFilePath)
- || config.isPersistFlatUseSnapshotEnabled) {
+ || !config.isPersistFlatUseSnapshotEnabled) {
newTableDS(tableRef)
} else {
sparkSession.read.parquet(snapshotResFilePath.toString).alias(tableRef.getAlias)
[kylin] 32/38: KYLIN-5523 [FOLLOW UP] expand the filter condition expression to ensure compatibility
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit cb3f4e604f1caa8d13de7987912d5638d408bced
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Sat Mar 4 21:06:08 2023 +0800
KYLIN-5523 [FOLLOW UP] expand the filter condition expression to ensure compatibility
---
.../kylin/metadata/model/tool/CalciteParser.java | 10 ++
.../kylin/rest/service/ModelSemanticHelper.java | 23 ++--
.../apache/kylin/rest/service/ModelService.java | 27 ++---
.../service/ModelServiceSemanticUpdateTest.java | 55 +++++++--
.../kylin/rest/service/ModelServiceTest.java | 18 +--
.../org/apache/kylin/query/util/EscapeParser.jj | 42 ++++++-
.../org/apache/kylin/query/util/EscapeDialect.java | 4 +
.../apache/kylin/query/util/EscapeFunction.java | 16 +++
.../org/apache/kylin/query/util/PushDownUtil.java | 17 ++-
.../org/apache/kylin/query/util/QueryUtil.java | 15 +++
.../query/util/EscapeTransformerCalciteTest.java | 22 ++++
.../query/util/EscapeTransformerSparkSqlTest.java | 11 ++
.../apache/kylin/query/util/PushDownUtilTest.java | 8 +-
.../org/apache/kylin/query/util/QueryUtilTest.java | 10 ++
.../engine/spark/builder/CreateFlatTable.scala | 123 +--------------------
.../kylin/engine/spark/job/FlatTableHelper.scala | 2 -
.../job/stage/build/FlatTableAndDictBase.scala | 13 ++-
.../spark/smarter/IndexDependencyParser.scala | 34 +++---
18 files changed, 256 insertions(+), 194 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
index 16c6e85ba6..5a8f0fcf0e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -29,11 +29,13 @@ import java.util.concurrent.TimeUnit;
import org.apache.calcite.avatica.util.Casing;
import org.apache.calcite.avatica.util.Quoting;
+import org.apache.calcite.config.NullCollation;
import org.apache.calcite.sql.SqlDialect;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.dialect.HiveSqlDialect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
@@ -58,6 +60,14 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class CalciteParser {
+ /**
+ * Overwrite {@link HiveSqlDialect#DEFAULT} with backtick quote.
+ */
+ public static final HiveSqlDialect HIVE_SQL_DIALECT = new HiveSqlDialect(
+ EMPTY_CONTEXT.withDatabaseProduct(SqlDialect.DatabaseProduct.HIVE) //
+ .withNullCollation(NullCollation.LOW) //
+ .withIdentifierQuoteString(Quoting.BACK_TICK.string));
+
private CalciteParser() {
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index df14faaf5a..38307a881c 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -39,7 +39,6 @@ import java.util.stream.Collectors;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.dialect.CalciteSqlDialect;
-import org.apache.calcite.sql.dialect.HiveSqlDialect;
import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -97,6 +96,7 @@ import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
import org.apache.kylin.query.util.PushDownUtil;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.request.ModelRequest;
import org.apache.kylin.rest.response.BuildIndexResponse;
import org.apache.kylin.rest.response.SimplifiedMeasure;
@@ -436,12 +436,12 @@ public class ModelSemanticHelper extends BasicService {
.forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
model.getComputedColumnDescs().forEach(x -> changeTableAlias(x, oldAliasName, newAliasName));
- String filterCondition = model.getFilterCondition();
- if (StringUtils.isNotEmpty(filterCondition)) {
- SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAliasName, newAliasName);
- SqlNode sqlNode = CalciteParser.getExpNode(filterCondition);
- sqlNode.accept(modifyAlias);
- String newFilterCondition = sqlNode.toSqlString(CalciteSqlDialect.DEFAULT, true).toString();
+ if (StringUtils.isNotBlank(model.getFilterCondition())) {
+ String expr = QueryUtil.adaptCalciteSyntax(model.getFilterCondition());
+ SqlNode sqlNode = CalciteParser.getExpNode(expr);
+ sqlNode.accept(new ModifyTableNameSqlVisitor(oldAliasName, newAliasName));
+
+ String newFilterCondition = sqlNode.toSqlString(CalciteParser.HIVE_SQL_DIALECT).toString();
model.setFilterCondition(newFilterCondition);
}
}
@@ -451,7 +451,7 @@ public class ModelSemanticHelper extends BasicService {
SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias);
SqlNode sqlNode = CalciteParser.getExpNode(computedColumnDesc.getExpression());
sqlNode.accept(modifyAlias);
- computedColumnDesc.setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString());
+ computedColumnDesc.setExpression(sqlNode.toSqlString(CalciteSqlDialect.DEFAULT).toString());
}
private Map<String, String> getAliasTransformMap(NDataModel originModel, NDataModel expectModel) {
@@ -868,7 +868,7 @@ public class ModelSemanticHelper extends BasicService {
originModel.getEffectiveMeasures().keySet());
}
- public boolean isFilterConditonNotChange(String oldFilterCondition, String newFilterCondition) {
+ public boolean isFilterConditionNotChange(String oldFilterCondition, String newFilterCondition) {
oldFilterCondition = oldFilterCondition == null ? "" : oldFilterCondition;
newFilterCondition = newFilterCondition == null ? "" : newFilterCondition;
return StringUtils.trim(oldFilterCondition).equals(StringUtils.trim(newFilterCondition));
@@ -903,7 +903,7 @@ public class ModelSemanticHelper extends BasicService {
return isDifferent(originModel.getPartitionDesc(), newModel.getPartitionDesc())
|| !Objects.equals(originModel.getRootFactTable(), newModel.getRootFactTable())
|| !originModel.getJoinsGraph().match(newModel.getJoinsGraph(), Maps.newHashMap())
- || !isFilterConditonNotChange(originModel.getFilterCondition(), newModel.getFilterCondition())
+ || !isFilterConditionNotChange(originModel.getFilterCondition(), newModel.getFilterCondition())
|| !isMultiPartitionDescSame(originModel.getMultiPartitionDesc(), newModel.getMultiPartitionDesc())
|| !isAntiFlattenableSame(originModel.getJoinTables(), newModel.getJoinTables());
}
@@ -977,7 +977,6 @@ public class ModelSemanticHelper extends BasicService {
return SourceFactory.getSource(tableDesc).getSegmentRange(start, end);
}
-
private void handleDatePartitionColumn(NDataModel newModel, NDataflowManager dataflowManager, NDataflow df,
String modelId, String project, String start, String end) {
// from having partition to no partition
@@ -986,7 +985,7 @@ public class ModelSemanticHelper extends BasicService {
Lists.newArrayList(SegmentRange.TimePartitionedSegmentRange.createInfinite()));
return;
}
- // change partition column and from no partition to having partition
+ // change partition column and from no partition to having partition
if (StringUtils.isNotEmpty(start) && StringUtils.isNotEmpty(end)) {
dataflowManager.fillDfManually(df,
Lists.newArrayList(getSegmentRangeByModel(project, modelId, start, end)));
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index d0073de148..a7b83f9ceb 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -199,6 +199,7 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.streaming.KafkaConfig;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
import org.apache.kylin.rest.request.ModelConfigRequest;
@@ -1305,7 +1306,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public String getModelSql(String modelId, String project) {
aclEvaluate.checkProjectReadPermission(project);
NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
- return PushDownUtil.generateFlatTableSql(model, project, false);
+ return PushDownUtil.generateFlatTableSql(model, false);
}
public List<RelatedModelResponse> getRelateModels(String project, String table, String modelId) {
@@ -1742,7 +1743,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
if (prjInstance.getSourceType() == ISourceAware.ID_SPARK
&& model.getModelType() == NDataModel.ModelType.BATCH) {
SparkSession ss = SparderEnv.getSparkSession();
- String flatTableSql = PushDownUtil.generateFlatTableSql(model, project, false);
+ String flatTableSql = PushDownUtil.generateFlatTableSql(model, false);
QueryParams queryParams = new QueryParams(project, flatTableSql, "default", false);
queryParams.setKylinConfig(prjInstance.getConfig());
queryParams.setAclInfo(AclPermissionUtil.createAclInfo(project, getCurrentUserGroups()));
@@ -3622,23 +3623,23 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
/**
- * massage and update model filter condition
- * 1. expand computed columns
- * 2. add missing identifier quotes
- * 3. add missing table identifiers
- *
- * @param model
+ * Convert model filter condition:
+ * 1. transform special function and backtick
+ * 2. use Calcite to add table name
+ * 3. massage for push-down
+ * 4. update the filter condition
*/
void massageModelFilterCondition(final NDataModel model) {
- if (StringUtils.isEmpty(model.getFilterCondition())) {
+ String filterCondition = model.getFilterCondition();
+ if (StringUtils.isBlank(filterCondition)) {
return;
}
- String filterConditionWithTableName = addTableNameIfNotExist(model.getFilterCondition(), model);
+ filterCondition = QueryUtil.adaptCalciteSyntax(filterCondition);
+ String newFilterCondition = addTableNameIfNotExist(filterCondition, model);
QueryContext.AclInfo aclInfo = AclPermissionUtil.createAclInfo(model.getProject(), getCurrentUserGroups());
- // validate as soon as possible
- PushDownUtil.massageExpression(model, model.getProject(), model.getFilterCondition(), aclInfo);
- model.setFilterCondition(filterConditionWithTableName);
+ String massaged = PushDownUtil.massageExpression(model, model.getProject(), newFilterCondition, aclInfo);
+ model.setFilterCondition(massaged);
}
@VisibleForTesting
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java
index 9fe9ea8aaf..b65770e577 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java
@@ -21,6 +21,8 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.SIMPLIFIED_
import static org.hamcrest.Matchers.is;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -38,6 +40,7 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.cube.model.SelectRule;
import org.apache.kylin.engine.spark.job.ExecutableAddCuboidHandler;
import org.apache.kylin.engine.spark.job.NSparkCubingJob;
@@ -1659,17 +1662,47 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testIsFilterConditonNotChange() {
- Assert.assertTrue(semanticService.isFilterConditonNotChange(null, null));
- Assert.assertTrue(semanticService.isFilterConditonNotChange("", null));
- Assert.assertTrue(semanticService.isFilterConditonNotChange(null, " "));
- Assert.assertTrue(semanticService.isFilterConditonNotChange(" ", ""));
- Assert.assertTrue(semanticService.isFilterConditonNotChange("", " "));
- Assert.assertTrue(semanticService.isFilterConditonNotChange("A=8", " A=8 "));
-
- Assert.assertFalse(semanticService.isFilterConditonNotChange(null, "null"));
- Assert.assertFalse(semanticService.isFilterConditonNotChange("", "null"));
- Assert.assertFalse(semanticService.isFilterConditonNotChange("A=8", "A=9"));
+ public void testUpdateModelColumnForTableAliasModify()
+ throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ NDataModel testModel = getTestModel();
+ Map<String, String> map = Maps.newHashMap();
+ map.put("TEST_ORDER", "TEST_ORDER1");
+ testModel.setFilterCondition("`TEST_ORDER`.`ORDER_ID` > 1");
+ ModelSemanticHelper semanticHelper = new ModelSemanticHelper();
+ Class<? extends ModelSemanticHelper> clazz = semanticHelper.getClass();
+ Method method = clazz.getDeclaredMethod("updateModelColumnForTableAliasModify", NDataModel.class, Map.class);
+ Unsafe.changeAccessibleObject(method, true);
+ method.invoke(semanticHelper, testModel, map);
+ Assert.assertEquals("`TEST_ORDER1`.`ORDER_ID` > 1", testModel.getFilterCondition());
+ Unsafe.changeAccessibleObject(method, false);
+ }
+
+ @Test
+ public void testChangeTableAlias() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ ComputedColumnDesc cc = new ComputedColumnDesc();
+ cc.setExpression("\"TEST_ORDER\".\"ORDER_ID\" + 1");
+ ModelSemanticHelper semanticHelper = new ModelSemanticHelper();
+ Class<? extends ModelSemanticHelper> clazz = semanticHelper.getClass();
+ Method method = clazz.getDeclaredMethod("changeTableAlias", ComputedColumnDesc.class, String.class,
+ String.class);
+ Unsafe.changeAccessibleObject(method, true);
+ method.invoke(semanticHelper, cc, "TEST_ORDER", "TEST_ORDER1");
+ Assert.assertEquals("\"TEST_ORDER1\".\"ORDER_ID\" + 1", cc.getExpression());
+ Unsafe.changeAccessibleObject(method, false);
+ }
+
+ @Test
+ public void testIsFilterConditionNotChange() {
+ Assert.assertTrue(semanticService.isFilterConditionNotChange(null, null));
+ Assert.assertTrue(semanticService.isFilterConditionNotChange("", null));
+ Assert.assertTrue(semanticService.isFilterConditionNotChange(null, " "));
+ Assert.assertTrue(semanticService.isFilterConditionNotChange(" ", ""));
+ Assert.assertTrue(semanticService.isFilterConditionNotChange("", " "));
+ Assert.assertTrue(semanticService.isFilterConditionNotChange("A=8", " A=8 "));
+
+ Assert.assertFalse(semanticService.isFilterConditionNotChange(null, "null"));
+ Assert.assertFalse(semanticService.isFilterConditionNotChange("", "null"));
+ Assert.assertFalse(semanticService.isFilterConditionNotChange("A=8", "A=9"));
}
@Test
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 58971f625c..1e3e316bd8 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -3266,8 +3266,8 @@ public class ModelServiceTest extends SourceTestCase {
modelRequest.getPartitionDesc().setPartitionDateFormat("yyyy-MM-dd");
String filterCond = "trans_id = 0 and TEST_KYLIN_FACT.order_id < 100 and DEAL_AMOUNT > 123";
- String expectedFilterCond = "(((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) "
- + "AND (\"TEST_KYLIN_FACT\".\"ORDER_ID\" < 100)) AND (\"TEST_KYLIN_FACT\".\"DEAL_AMOUNT\" > 123))";
+ String expectedFilterCond = "(((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) AND (`TEST_KYLIN_FACT`.`ORDER_ID` < 100)) "
+ + "AND ((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))";
modelRequest.setFilterCondition(filterCond);
val newModel = modelService.createModel(modelRequest.getProject(), modelRequest);
@@ -3291,7 +3291,7 @@ public class ModelServiceTest extends SourceTestCase {
modelRequest.setUuid(null);
String filterCond = "\"day\" = 0 and \"123TABLE\".\"day#\" = 1 and \"中文列\" = 1";
- String expectedFilterCond = "(((\"123TABLE\".\"DAY\" = 0) AND (\"123TABLE\".\"day#\" = 1)) AND (\"123TABLE\".\"中文列\" = 1))";
+ String expectedFilterCond = "(((`123TABLE`.`DAY` = 0) AND (`123TABLE`.`day#` = 1)) AND (`123TABLE`.`中文列` = 1))";
modelRequest.setFilterCondition(filterCond);
val newModel = modelService.createModel(modelRequest.getProject(), modelRequest);
@@ -3540,8 +3540,9 @@ public class ModelServiceTest extends SourceTestCase {
String originSql = "trans_id = 0 and TEST_KYLIN_FACT.order_id < 100 and DEAL_AMOUNT > 123";
model.setFilterCondition(originSql);
modelService.massageModelFilterCondition(model);
- Assert.assertEquals("(((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) AND (\"TEST_KYLIN_FACT\".\"ORDER_ID\" < 100)) "
- + "AND (\"TEST_KYLIN_FACT\".\"DEAL_AMOUNT\" > 123))", model.getFilterCondition());
+ Assert.assertEquals("(((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) "
+ + "AND (`TEST_KYLIN_FACT`.`ORDER_ID` < 100)) AND ((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))",
+ model.getFilterCondition());
}
@Test
@@ -3573,7 +3574,7 @@ public class ModelServiceTest extends SourceTestCase {
final NDataModel model1 = modelManager.getDataModelDesc(modelId);
model1.setFilterCondition("TIMESTAMPDIFF(DAY, CURRENT_DATE, TEST_KYLIN_FACT.\"CURRENT_DATE\") >= 0");
modelService.massageModelFilterCondition(model1);
- Assert.assertEquals("(TIMESTAMPDIFF(DAY, CURRENT_DATE(), \"TEST_KYLIN_FACT\".\"CURRENT_DATE\") >= 0)",
+ Assert.assertEquals("(TIMESTAMPDIFF('DAY', CURRENT_DATE(), `TEST_KYLIN_FACT`.`CURRENT_DATE`) >= 0)",
model1.getFilterCondition());
}
@@ -3589,8 +3590,9 @@ public class ModelServiceTest extends SourceTestCase {
String originSql = "trans_id = 0 and TEST_ORDER.order_id < 100 and DEAL_AMOUNT > 123";
model.setFilterCondition(originSql);
modelService.massageModelFilterCondition(model);
- Assert.assertEquals("(((\"TEST_KYLIN_FACT\".\"TRANS_ID\" = 0) AND (\"TEST_ORDER\".\"ORDER_ID\" < 100)) "
- + "AND (\"TEST_KYLIN_FACT\".\"DEAL_AMOUNT\" > 123))", model.getFilterCondition());
+ Assert.assertEquals("(((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) "
+ + "AND (`TEST_ORDER`.`ORDER_ID` < 100)) AND ((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))",
+ model.getFilterCondition());
}
@Test
diff --git a/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/EscapeParser.jj b/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/EscapeParser.jj
index d09a3cfb32..f253afc12c 100644
--- a/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/EscapeParser.jj
+++ b/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/EscapeParser.jj
@@ -13,7 +13,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Scanner;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -127,6 +127,8 @@ TOKEN :
| < #EXPONENT: ["e","E"] (["+","-"])? (["0"-"9"])+ >
| < DOT : "." >
| < CEIL : "CEIL" >
+| < CEIL_DATETIME : "CEIL_DATETIME" >
+| < FLOOR_DATETIME : "FLOOR_DATETIME" >
| < FLOOR : "FLOOR" >
| < TO : "TO" >
| < SUBSTRING : "SUBSTRING" >
@@ -237,6 +239,7 @@ String Expression() :
| innerString = CastExpression()
| innerString = ExtractExpression()
| innerString = CeilFloorExpress()
+ | innerString = CeilFloorDatetimeExpress()
| innerString = ParenExpress()
| innerString = QuotedString()
| innerString = DoubleQuotedString()
@@ -662,6 +665,42 @@ String CeilFloorExpress() :
}
}
+String CeilFloorDatetimeExpress() :
+{
+ String functionName;
+ List <String> parameters = Lists.newArrayList();
+ ImmutableSet<String> timeunitSet = ImmutableSet.of("YEAR", "QUARTER", "MONTH", "WEEK",
+ "DAY", "HOUR", "MINUTE", "SECOND");
+}
+{
+ ( < CEIL_DATETIME> | < FLOOR_DATETIME> )
+ {
+ functionName = getToken(0).image;
+ }
+ (< SPACE >)* <LPAREN> (< SPACE >)*
+ {
+ parameters.add(ParameterExpression().trim());
+ }
+ (< SPACE >)*
+ (
+ <COMMA> (<SPACE>)*
+ {
+ String timeUnit = StringUtils.trim(ParameterExpression());
+ String unitToValidate = StringUtils.remove(timeUnit, '\'');
+ if (!timeunitSet.contains(StringUtils.upperCase(unitToValidate))) {
+ throw new IllegalStateException(String.format(Locale.ROOT, CEIL_FLOOR_EXCEPTION_MSG,
+ functionName, unitToValidate, String.join(", ", timeunitSet)));
+ }
+ parameters.add(timeUnit);
+ }
+ (<SPACE>)*
+ )?
+ <RPAREN>
+ {
+ return dialect.transformFN(functionName, parameters.toArray(new String [ 0 ]));
+ }
+}
+
String PiFunction() :
{
String function;
@@ -766,6 +805,7 @@ String ParameterExpression() :
| innerString = TsDiffOrAddExpression()
| innerString = ExtractExpression()
| innerString = CeilFloorExpress()
+ | innerString = CeilFloorDatetimeExpress()
| innerString = Numeric()
| innerString = Hint()
| innerString = CubePriority()
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeDialect.java b/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeDialect.java
index 0e5a1b34c9..11561ea0aa 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeDialect.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeDialect.java
@@ -32,6 +32,8 @@ public abstract class EscapeDialect {
private static final String FN_WEEK = "WEEK";
private static final String FN_CEIL = "CEIL";
private static final String FN_FLOOR = "FLOOR";
+ private static final String FN_CEIL_DT = "CEIL_DATETIME";
+ private static final String FN_FLOOR_DT = "FLOOR_DATETIME";
private static final String FN_SUBSTR = "SUBSTR";
private static final String FN_SUBSTRING = "SUBSTRING";
private static final String FN_ASCII = "ASCII";
@@ -74,6 +76,8 @@ public abstract class EscapeDialect {
register(FN_WEEK, FnConversion.WEEK_CALCITE);
register(FN_CEIL, FnConversion.CEIL);
register(FN_FLOOR, FnConversion.FLOOR);
+ register(FN_CEIL_DT, FnConversion.CEIL_DT);
+ register(FN_FLOOR_DT, FnConversion.FLOOR_DT);
register(FN_SUBSTR, FnConversion.SUSTR);
register(FN_SUBSTRING, FnConversion.SUSTRING);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeFunction.java b/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeFunction.java
index 09d8947a99..e297631bed 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeFunction.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeFunction.java
@@ -21,6 +21,8 @@ package org.apache.kylin.query.util;
import java.util.List;
import java.util.Locale;
+import org.apache.commons.lang3.StringUtils;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -327,6 +329,20 @@ public class EscapeFunction {
return normalFN("FLOOR", newArgs);
}),
+ // such as: ceil_datetime(col, 'year') => ceil(col to year)
+ CEIL_DT(args -> {
+ Preconditions.checkArgument(args.length == 2, EscapeFunction.CEIL_EXCEPTION_MSG);
+ String[] newArgs = new String[] { args[0] + " to " + StringUtils.remove(args[1], '\'') };
+ return normalFN("CEIL", newArgs);
+ }),
+
+ // such as: floor_datetime(col, 'year') => floor(col to year)
+ FLOOR_DT(args -> {
+ Preconditions.checkArgument(args.length == 2, EscapeFunction.FLOOR_EXCEPTION_MSG);
+ String[] newArgs = new String[] { args[0] + " to " + StringUtils.remove(args[1], '\'') };
+ return normalFN("FLOOR", newArgs);
+ }),
+
// tableau func
SPACE(args -> {
checkArgs(args, 1);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 4f7f467e17..60c02d73dc 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -173,7 +173,11 @@ public class PushDownUtil {
public static String massageExpression(NDataModel model, String project, String expression,
QueryContext.AclInfo aclInfo) {
- String ccSql = expandComputedColumnExp(model, project, expression);
+ if (StringUtils.isBlank(expression)) {
+ return "";
+ }
+
+ String ccSql = expandComputedColumnExp(model, project, StringHelper.backtickToDoubleQuote(expression));
QueryParams queryParams = new QueryParams(project, ccSql, PushDownUtil.DEFAULT_SCHEMA, false);
queryParams.setKylinConfig(NProjectManager.getProjectConfig(project));
queryParams.setAclInfo(aclInfo);
@@ -223,9 +227,10 @@ public class PushDownUtil {
}
/**
- * This method is currently only used for verifying the flat-table generated by the model.
+ * Generate flat-table SQL which can be parsed by Apache Calcite.
+ * If querying push-down is required, please use it in conjunction with {@link PushDownUtil#massagePushDownSql}.
*/
- public static String generateFlatTableSql(NDataModel model, String project, boolean singleLine) {
+ public static String generateFlatTableSql(NDataModel model, boolean singleLine) {
String sep = singleLine ? " " : "\n";
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("SELECT ").append(sep);
@@ -237,7 +242,7 @@ public class PushDownUtil {
String allColStr = tblColRefs.stream() //
.filter(colRef -> !colRef.getColumnDesc().isComputedColumn()) //
.map(colRef -> {
- String s = colRef.getTableAlias() + UNDER_LINE + colRef.getName();
+ String s = colRef.getTableAlias() + PushDownUtil.UNDER_LINE + colRef.getName();
String colName = StringHelper.doubleQuote(s);
return colRef.getDoubleQuoteExp() + " as " + colName + sep;
}).collect(Collectors.joining(", "));
@@ -251,8 +256,8 @@ public class PushDownUtil {
sqlBuilder.append("WHERE ").append(sep);
sqlBuilder.append("1 = 1").append(sep);
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
- massageExpression(model, project, model.getFilterCondition(), null);
- sqlBuilder.append(" AND (").append(model.getFilterCondition()).append(") ").append(sep);
+ String filterConditionWithCalciteFormat = QueryUtil.adaptCalciteSyntax(model.getFilterCondition());
+ sqlBuilder.append(" AND (").append(filterConditionWithCalciteFormat).append(") ").append(sep);
}
return new EscapeTransformer().transform(sqlBuilder.toString());
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 6459d29a10..77c7b2784d 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -34,6 +34,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinTimeoutException;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.BigQueryThresholdUpdater;
import org.apache.kylin.query.IQueryTransformer;
@@ -60,6 +61,7 @@ public class QueryUtil {
private static final String SEMI_COLON = ";";
public static final String JDBC = "jdbc";
private static final Map<String, IQueryTransformer> QUERY_TRANSFORMER_MAP = Maps.newConcurrentMap();
+ private static final EscapeTransformer ESCAPE_TRANSFORMER = new EscapeTransformer();
static {
String[] classNames = KylinConfig.getInstanceFromEnv().getQueryTransformers();
@@ -76,6 +78,19 @@ public class QueryUtil {
private QueryUtil() {
}
+ /**
+ * Convert special functions in the input string into the counterpart
+ * which can be parsed by Apache Calcite.
+ */
+ public static String adaptCalciteSyntax(String str) {
+ if (StringUtils.isBlank(str)) {
+ return str;
+ }
+ String transformed = ESCAPE_TRANSFORMER.transform(str);
+ transformed = StringHelper.backtickToDoubleQuote(transformed);
+ return transformed;
+ }
+
public static boolean isSelectStatement(String sql) {
String sql1 = sql.toLowerCase(Locale.ROOT);
sql1 = removeCommentInSql(sql1);
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerCalciteTest.java b/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerCalciteTest.java
index baefd0fecd..25bc62e62a 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerCalciteTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerCalciteTest.java
@@ -179,6 +179,28 @@ public class EscapeTransformerCalciteTest {
Assert.assertEquals(expectedSQL, transformedSQL);
}
+ @Test
+ public void ceilFloorDtTest() {
+ String originSQL = "select ceil_datetime('2012-02-02 00:23:23', 'year'), ceil(floor_datetime(col, 'hour') to day)";
+ String expectedSQL = "select CEIL('2012-02-02 00:23:23' to year), CEIL(FLOOR(col to hour) to day)";
+
+ String transformedSQL = transformer.transform(originSQL);
+ Assert.assertEquals(expectedSQL, transformedSQL);
+ }
+
+ @Test
+ public void testFailToTransformCeilFloorDt() {
+ {
+ String origin = "select ceil_datetime('2012-02-02 00:23:23')";
+ Assert.assertEquals(origin, transformer.transform(origin));
+ }
+
+ {
+ String origin = "select floor_datetime('2012-02-02 00:23:23')";
+ Assert.assertEquals(origin, transformer.transform(origin));
+ }
+ }
+
@Test
public void testCeilFloorQuery() {
String originSql = "SELECT {FN WEEK(CEIL( FLOOR(\t TIME2 TO HOUR ) TO DAY )) }, FLOOR(SELLER_ID), CEIL(SELLER_ID) FROM TEST_MEASURE";
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerSparkSqlTest.java b/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerSparkSqlTest.java
index 8523f0b608..ad9701d1f9 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerSparkSqlTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/EscapeTransformerSparkSqlTest.java
@@ -157,6 +157,17 @@ public class EscapeTransformerSparkSqlTest {
Assert.assertEquals(expectedSQL, transformedSQL);
}
+ @Test
+ public void ceilFloorDtTest() {
+ // The case ceil(floor_datetime(col, 'hour') to day) won't happen,
+ // so just give follow example to illustrate the normal case won't be replaced.
+ String originSQL = "select ceil_datetime('2012-02-02 00:23:23', 'year'), ceil_datetime(floor_datetime(col, 'hour'), 'day')";
+ String expectedSQL = "select ceil_datetime('2012-02-02 00:23:23', 'year'), ceil_datetime(floor_datetime(col, 'hour'), 'day')";
+
+ String transformedSQL = transformer.transform(originSQL);
+ Assert.assertEquals(expectedSQL, transformedSQL);
+ }
+
@Test
public void testCeilFloorQuery() {
String originSql = "SELECT {FN WEEK(CEIL( FLOOR(\t TIME2 TO HOUR ) TO DAY )) }, FLOOR(SELLER_ID), CEIL(SELLER_ID) FROM TEST_MEASURE";
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index 379af78e8e..5b91a8cef2 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -167,7 +167,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ "WHERE\n" //
+ "1 = 1";
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, project, false));
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, false));
}
@Test
@@ -204,7 +204,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ "WHERE\n" //
+ "1 = 1";
NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
}
@@ -239,7 +239,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ "1 = 1\n" //
+ " AND (SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china')";
NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
}
@Test
@@ -272,7 +272,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ "1 = 1\n" //
+ " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and cc1 = 'china')";
NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
+ Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
}
private void updateModelToAddCC(String project, NDataModel model) {
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index 0b2120263f..022f457308 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -423,6 +423,16 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
newSql2);
}
+ @Test
+ public void testAdaptCalciteSyntax() {
+ Assert.assertEquals("a\"b(", QueryUtil.adaptCalciteSyntax("a\"b("));
+ Assert.assertEquals(" ", QueryUtil.adaptCalciteSyntax(" "));
+ Assert.assertEquals("", QueryUtil.adaptCalciteSyntax(""));
+ Assert.assertEquals("CEIL(col to year)", QueryUtil.adaptCalciteSyntax("ceil_datetime(col, 'year')"));
+ Assert.assertEquals("CEIL(\"t\".\"col\" to year)",
+ QueryUtil.adaptCalciteSyntax("ceil_datetime(`t`.`col`, 'year')"));
+ }
+
@Test
public void testLimitOffsetMatch() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
index fdb4d87cd2..bdcd74ea3c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
@@ -18,28 +18,27 @@
package org.apache.kylin.engine.spark.builder
-import java.util
-import com.google.common.collect.Sets
-import org.apache.kylin.engine.spark.builder.DFBuilderHelper.{ENCODE_SUFFIX, _}
+import java.util.Locale
+
+import org.apache.commons.lang3.StringUtils
+import org.apache.kylin.engine.spark.builder.DFBuilderHelper._
import org.apache.kylin.engine.spark.job.NSparkCubingUtil._
import org.apache.kylin.engine.spark.job.{FlatTableHelper, TableMetaManager}
import org.apache.kylin.engine.spark.utils.SparkDataSource._
import org.apache.kylin.engine.spark.utils.{LogEx, LogUtils}
import org.apache.kylin.metadata.cube.cuboid.NSpanningTree
import org.apache.kylin.metadata.cube.model.{NCubeJoinedFlatTableDesc, NDataSegment}
-import org.apache.kylin.metadata.model.NDataModel
-import org.apache.commons.lang3.StringUtils
import org.apache.kylin.metadata.model._
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.{Dataset, Row, SparkSession}
-import java.util.Locale
import scala.collection.JavaConverters._
import scala.collection.mutable
-import scala.collection.mutable.ListBuffer
import scala.collection.parallel.ForkJoinTaskSupport
import scala.concurrent.forkjoin.ForkJoinPool
+import com.google.common.collect.Sets
+
@Deprecated
class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
@@ -323,116 +322,6 @@ object CreateFlatTable extends LogEx {
newdf
}
- /*
- * Convert IJoinedFlatTableDesc to SQL statement
- */
- def generateSelectDataStatement(flatDesc: IJoinedFlatTableDesc,
- singleLine: Boolean,
- skipAs: Array[String]): String = {
- val sep: String = {
- if (singleLine) " "
- else "\n"
- }
- val skipAsList = {
- if (skipAs == null) ListBuffer.empty[String]
- else skipAs.toList
- }
- val sql: StringBuilder = new StringBuilder
- sql.append("SELECT" + sep)
- for (i <- 0 until flatDesc.getAllColumns.size()) {
- val col: TblColRef = flatDesc.getAllColumns.get(i)
- sql.append(",")
- val colTotalName: String =
- String.format(Locale.ROOT, "%s.%s", col.getTableRef.getTableName, col.getName)
- if (skipAsList.contains(colTotalName)) {
- sql.append(col.getExpressionInSourceDB + sep)
- } else {
- sql.append(col.getExpressionInSourceDB + " as " + colName(col) + sep)
- }
- }
- appendJoinStatement(flatDesc, sql, singleLine)
- appendWhereStatement(flatDesc, sql, singleLine)
- sql.toString
- }
-
- def appendJoinStatement(flatDesc: IJoinedFlatTableDesc,
- sql: StringBuilder,
- singleLine: Boolean): Unit = {
- val sep: String =
- if (singleLine) " "
- else "\n"
- val dimTableCache: util.Set[TableRef] = Sets.newHashSet[TableRef]
- val model: NDataModel = flatDesc.getDataModel
- val rootTable: TableRef = model.getRootFactTable
- sql.append(
- "FROM " + flatDesc.getDataModel.getRootFactTable.getTableIdentity + " as " + rootTable.getAlias + " " + sep)
- for (lookupDesc <- model.getJoinTables.asScala) {
- val join: JoinDesc = lookupDesc.getJoin
- if (join != null && join.getType == "" == false) {
- val joinType: String = join.getType.toUpperCase(Locale.ROOT)
- val dimTable: TableRef = lookupDesc.getTableRef
- if (!dimTableCache.contains(dimTable)) {
- val pk: Array[TblColRef] = join.getPrimaryKeyColumns
- val fk: Array[TblColRef] = join.getForeignKeyColumns
- if (pk.length != fk.length) {
- throw new RuntimeException(
- "Invalid join condition of lookup table:" + lookupDesc)
- }
- sql.append(
- joinType + " JOIN " + dimTable.getTableIdentity + " as " + dimTable.getAlias + sep)
- sql.append("ON ")
- var i: Int = 0
- while ( {
- i < pk.length
- }) {
- if (i > 0) sql.append(" AND ")
- sql.append(
- fk(i).getExpressionInSourceDB + " = " + pk(i).getExpressionInSourceDB)
-
- {
- i += 1
- i - 1
- }
- }
- sql.append(sep)
- dimTableCache.add(dimTable)
- }
- }
- }
- }
-
- private def appendWhereStatement(flatDesc: IJoinedFlatTableDesc,
- sql: StringBuilder,
- singleLine: Boolean): Unit = {
- val sep: String =
- if (singleLine) " "
- else "\n"
- val whereBuilder: StringBuilder = new StringBuilder
- whereBuilder.append("WHERE 1=1")
- val model: NDataModel = flatDesc.getDataModel
- if (StringUtils.isNotEmpty(model.getFilterCondition)) {
- whereBuilder
- .append(" AND (")
- .append(model.getFilterCondition)
- .append(") ")
- }
- val partDesc: PartitionDesc = model.getPartitionDesc
- val segRange: SegmentRange[_ <: Comparable[_]] = flatDesc.getSegRange
- if (flatDesc.getSegment != null && partDesc != null
- && partDesc.getPartitionDateColumn != null && segRange != null && !segRange.isInfinite) {
- val builder =
- flatDesc.getDataModel.getPartitionDesc.getPartitionConditionBuilder
- if (builder != null) {
- whereBuilder.append(" AND (")
- whereBuilder.append(
- builder
- .buildDateRangeCondition(partDesc, flatDesc.getSegment, segRange))
- whereBuilder.append(")" + sep)
- }
-
- sql.append(whereBuilder.toString)
- }
- }
def colName(col: TblColRef): String = {
col.getTableAlias + "_" + col.getName
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
index 5964c661a6..3ba93d705e 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
@@ -21,7 +21,6 @@ package org.apache.kylin.engine.spark.job
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.engine.spark.builder.CreateFlatTable.replaceDot
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc
-import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.internal.Logging
import org.apache.spark.sql.{Dataset, Row}
@@ -56,7 +55,6 @@ object FlatTableHelper extends Logging {
if (StringUtils.isNotBlank(model.getFilterCondition)) {
var filterCond = model.getFilterCondition
- filterCond = PushDownUtil.massageExpression(model, model.getProject, filterCond, null);
if (needReplaceDot) filterCond = replaceDot(filterCond, model)
filterCond = s" (1=1) AND (" + filterCond + s")"
logInfo(s"Filter condition is $filterCond")
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 0599fc95ae..93df9cf9a4 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -18,7 +18,9 @@
package org.apache.kylin.engine.spark.job.stage.build
-import com.google.common.collect.Sets
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+import java.util.{Locale, Objects, Timer, TimerTask}
+
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.Path
import org.apache.kylin.common.util.HadoopUtil
@@ -40,7 +42,6 @@ import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBu
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.cube.planner.CostBasePlannerUtils
import org.apache.kylin.metadata.model._
-import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.sql.KapFunctions.dict_encode_v3
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, expr}
@@ -64,6 +65,8 @@ import scala.concurrent.duration.{Duration, MILLISECONDS}
import scala.concurrent.forkjoin.ForkJoinPool
import scala.util.{Failure, Success, Try}
+import com.google.common.collect.Sets
+
abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
private val dataSegment: NDataSegment,
private val buildParam: BuildParam)
@@ -318,12 +321,12 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
}
private def applyFilterCondition(originDS: Dataset[Row]): Dataset[Row] = {
- if (StringUtils.isBlank(dataModel.getFilterCondition)) {
+ val filterCondition = dataModel.getFilterCondition
+ if (StringUtils.isBlank(filterCondition)) {
logInfo(s"No available FILTER-CONDITION segment $segmentId")
return originDS
}
- val expression = PushDownUtil.massageExpression(dataModel, project, dataModel.getFilterCondition, null)
- val converted = replaceDot(expression, dataModel)
+ val converted = replaceDot(filterCondition, dataModel)
val condition = s" (1=1) AND ($converted)"
logInfo(s"Apply FILTER-CONDITION: $condition segment $segmentId")
originDS.where(condition)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
index 04234294c9..cb8229e705 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
@@ -17,7 +17,9 @@
*/
package org.apache.kylin.engine.spark.smarter
-import com.google.common.collect.{Lists, Maps, Sets}
+import java.util
+import java.util.Collections
+
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.engine.spark.job.NSparkCubingUtil
@@ -29,11 +31,11 @@ import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.{Dataset, Row, SparderEnv, SparkSession}
-import java.util
-import java.util.Collections
import scala.collection.JavaConverters._
import scala.collection.mutable
+import com.google.common.collect.{Lists, Maps, Sets}
+
class IndexDependencyParser(val model: NDataModel) {
private val ccTableNameAliasMap = Maps.newHashMap[String, util.Set[String]]
@@ -95,7 +97,7 @@ class IndexDependencyParser(val model: NDataModel) {
model.getEffectiveMeasures.get(id).getFunction.getParameters == null)
}
- private def getTableIdentitiesFromColumn(ref: TblColRef) = {
+ private def getTableIdentitiesFromColumn(ref: TblColRef): util.HashSet[String] = {
val desc = ref.getColumnDesc
if (desc.isComputedColumn) {
Sets.newHashSet(ccTableNameAliasMap.get(ref.getName))
@@ -172,18 +174,20 @@ class IndexDependencyParser(val model: NDataModel) {
result
}
- private def initFilterConditionTableNames(originDf: Dataset[Row], colFields: Array[StructField]): Unit =
- if (StringUtils.isNotEmpty(model.getFilterCondition)) {
- val whereDs = originDf.selectExpr(NSparkCubingUtil.convertFromDotWithBackTick(model.getFilterCondition.replace("\"", "`")))
- whereDs.schema.fields.foreach(whereField => {
- colFields.foreach(colField => {
- if (whereField.name.contains(colField.name)) {
- val tableName = colField.name.substring(0, colField.name.indexOf(NSparkCubingUtil.SEPARATOR))
- allTablesAlias.add(model.getTableNameMap.get(tableName).getAlias)
- }
- })
- })
+ private def initFilterConditionTableNames(originDf: Dataset[Row], colFields: Array[StructField]): Unit = {
+ if (StringUtils.isBlank(model.getFilterCondition)) {
+ return
}
+ val whereDs = originDf.selectExpr(NSparkCubingUtil.convertFromDotWithBackTick(model.getFilterCondition))
+ whereDs.schema.fields.foreach(whereField => {
+ colFields.foreach(colField => {
+ if (whereField.name.contains(colField.name)) {
+ val tableName = colField.name.substring(0, colField.name.indexOf(NSparkCubingUtil.SEPARATOR))
+ allTablesAlias.add(model.getTableNameMap.get(tableName).getAlias)
+ }
+ })
+ })
+ }
private def initPartitionColumnTableNames(): Unit = {
if (model.getPartitionDesc != null && model.getPartitionDesc.getPartitionDateColumnRef != null) {
[kylin] 23/38: KYLIN-5521 [FOLLOW UP] Fix JoinsGraph toString method StackOverFlow
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 625c4b6c7e86aa8402e450c085f8332b0f5fc8b2
Author: Jiale He <35...@users.noreply.github.com>
AuthorDate: Tue Feb 28 10:10:36 2023 +0800
KYLIN-5521 [FOLLOW UP] Fix JoinsGraph toString method StackOverFlow
---
.../apache/kylin/metadata/model/graph/JoinsGraph.java | 12 ++++++++++--
.../org/apache/kylin/metadata/model/JoinsGraphTest.java | 17 +++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
index 2029fd61ac..ee4efbf60b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
@@ -476,6 +476,10 @@ public class JoinsGraph implements Serializable {
return unmatched;
}
+ public List<Edge> getEdgesByFKSideWithoutSwap(TableRef table) {
+ return getEdgesByFKSide(table).stream().filter(e -> !e.isSwapJoin()).collect(Collectors.toList());
+ }
+
/**
* Root: TableRef[TEST_KYLIN_FACT]
* Edge: TableRef[TEST_KYLIN_FACT] LEFT JOIN TableRef[TEST_ORDER] ON [ORDER_ID] = [ORDER_ID]
@@ -487,7 +491,9 @@ public class JoinsGraph implements Serializable {
StringBuilder sb = new StringBuilder();
sb.append("Root: ").append(center);
// build next edges
- getEdgesByFKSide(center).forEach(e -> buildGraphStr(sb, e, 1));
+ for (Edge e : getEdgesByFKSideWithoutSwap(center)) {
+ buildGraphStr(sb, e, 1);
+ }
return sb.toString();
}
@@ -495,7 +501,9 @@ public class JoinsGraph implements Serializable {
sb.append(IntStream.range(0, indent).mapToObj(i -> " ")
.collect(Collectors.joining("", "\n", String.valueOf(edge))));
// build next edges
- getEdgesByFKSide(edge.pkSide()).forEach(e -> buildGraphStr(sb, e, indent + 1));
+ for (Edge e : getEdgesByFKSideWithoutSwap(edge.pkSide())) {
+ buildGraphStr(sb, e, indent + 1);
+ }
}
private <T> void addIfAbsent(List<T> edges, T edge) {
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
index 65a359e63f..59a946831c 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Supplier;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
@@ -368,4 +369,20 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
Object invoke = method.invoke(matcher, a, b);
Assert.assertEquals(true, invoke);
}
+
+ @Test
+ public void testToString() {
+ NDataModel modelDesc = NDataModelManager.getInstance(getTestConfig(), "default")
+ .getDataModelDescByAlias("nmodel_basic_inner");
+ JoinsGraph graph1 = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
+ .innerJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" }).build();
+ Assert.assertTrue(StringUtils.isNotEmpty(graph1.toString()));
+
+ JoinsGraph graph2 = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
+ .innerJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
+ .innerJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" })
+ .innerJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_COUNTRY" }, new String[] { "BUYER_COUNTRY.COUNTRY" })
+ .build();
+ Assert.assertTrue(StringUtils.isNotEmpty(graph2.toString()));
+ }
}
[kylin] 27/38: KYLIN-5523 [FOLLOW UP] Fix the error caused by using ceil/floor functions in model filtering conditions.
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 23bf641c017c2afe9d34eec0e81386e1e544062d
Author: Pengfei.Zhan <pe...@kyligence.io>
AuthorDate: Tue Feb 28 18:28:40 2023 +0800
KYLIN-5523 [FOLLOW UP] Fix the error caused by using ceil/floor functions in model filtering conditions.
---
.../src/main/java/org/apache/kylin/query/util/PushDownUtil.java | 4 ++--
.../src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index b902bfe3f5..2f0558d246 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -250,8 +250,8 @@ public class PushDownUtil {
sqlBuilder.append("WHERE ").append(sep);
sqlBuilder.append("1 = 1").append(sep);
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
- String filterCondition = massageExpression(model, project, model.getFilterCondition(), null);
- sqlBuilder.append(" AND (").append(filterCondition).append(") ").append(sep);
+ massageExpression(model, project, model.getFilterCondition(), null);
+ sqlBuilder.append(" AND (").append(model.getFilterCondition()).append(") ").append(sep);
}
return new EscapeTransformer().transform(sqlBuilder.toString());
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index fdc7dcf28c..804fd81ab7 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -237,7 +237,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ "WHERE\n" //
+ "1 = 1\n" //
- + " AND (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4) = 'china' and (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)) = 'china')";
+ + " AND (SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china')";
NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
}
@@ -270,7 +270,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ "WHERE\n" //
+ "1 = 1\n" //
- + " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and (SUBSTRING(`TEST_BANK_INCOME`.`COUNTRY`, 0, 4)) = 'china')";
+ + " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and cc1 = 'china')";
NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, project, false));
}
[kylin] 01/38: KYLIN-5521 fix JoinsGraph
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f66b85f0d286ce6b71f100b47da90288f63ab5b0
Author: Jiale He <ji...@gmail.com>
AuthorDate: Wed Feb 1 17:41:37 2023 +0800
KYLIN-5521 fix JoinsGraph
---
.../apache/kylin/metadata/model/JoinsGraph.java | 607 ---------------------
.../apache/kylin/metadata/model/NDataModel.java | 1 +
.../model/graph/DefaultJoinEdgeMatcher.java | 69 +++
.../apache/kylin/metadata/model/graph/Edge.java | 148 +++++
.../metadata/model/graph/IJoinEdgeMatcher.java | 27 +
.../kylin/metadata/model/graph/JoinsGraph.java | 506 +++++++++++++++++
.../metadata/model/util/ComputedColumnUtil.java | 4 +-
.../kylin/metadata/model/JoinsGraphTest.java | 89 ++-
.../kylin/metadata/model/MockJoinGraphBuilder.java | 1 +
.../apache/kylin/query/relnode/OLAPContext.java | 4 +-
.../kylin/query/routing/RealizationChooser.java | 2 +-
.../apache/kylin/query/util/QueryAliasMatcher.java | 5 +-
12 files changed, 819 insertions(+), 644 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
deleted file mode 100644
index 7df0ddbd6e..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
+++ /dev/null
@@ -1,607 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.metadata.model;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import lombok.Getter;
-import lombok.NonNull;
-import lombok.Setter;
-
-public class JoinsGraph implements Serializable {
-
- public class Edge implements Serializable {
-
- @Getter
- private final JoinDesc join;
- private final ColumnDesc[] leftCols;
- private final ColumnDesc[] rightCols;
- private final NonEquiJoinCondition nonEquiJoinCondition;
-
- private Edge(JoinDesc join) {
- this.join = join;
-
- leftCols = new ColumnDesc[join.getForeignKeyColumns().length];
- int i = 0;
- for (TblColRef colRef : join.getForeignKeyColumns()) {
- leftCols[i++] = colRef.getColumnDesc();
- }
-
- rightCols = new ColumnDesc[join.getPrimaryKeyColumns().length];
- i = 0;
- for (TblColRef colRef : join.getPrimaryKeyColumns()) {
- rightCols[i++] = colRef.getColumnDesc();
- }
-
- nonEquiJoinCondition = join.getNonEquiJoinCondition();
- }
-
- public boolean isJoinMatched(JoinDesc other) {
- return join.equals(other);
- }
-
- public boolean isNonEquiJoin() {
- return nonEquiJoinCondition != null;
- }
-
- public boolean isLeftJoin() {
- return !join.isLeftOrInnerJoin() && join.isLeftJoin();
- }
-
- public boolean isLeftOrInnerJoin() {
- return join.isLeftOrInnerJoin();
- }
-
- public boolean isInnerJoin() {
- return !join.isLeftOrInnerJoin() && join.isInnerJoin();
- }
-
- private TableRef left() {
- return join.getFKSide();
- }
-
- private TableRef right() {
- return join.getPKSide();
- }
-
- private boolean isFkSide(TableRef tableRef) {
- return join.getFKSide().equals(tableRef);
- }
-
- private boolean isPkSide(TableRef tableRef) {
- return join.getPKSide().equals(tableRef);
- }
-
- private TableRef other(TableRef tableRef) {
- if (left().equals(tableRef)) {
- return right();
- } else if (right().equals(tableRef)) {
- return left();
- }
- throw new IllegalArgumentException("table " + tableRef + " is not on the edge " + this);
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
-
- if (this.getClass() != that.getClass())
- return false;
-
- return joinEdgeMatcher.matches(this, (Edge) that);
- }
-
- @Override
- public int hashCode() {
- if (this.isLeftJoin()) {
- return Objects.hash(isLeftJoin(), leftCols, rightCols);
- } else {
- if (Arrays.hashCode(leftCols) < Arrays.hashCode(rightCols)) {
- return Objects.hash(isLeftJoin(), leftCols, rightCols);
- } else {
- return Objects.hash(isLeftJoin(), rightCols, leftCols);
- }
- }
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Edge: ");
- sb.append(left())
- .append(isLeftJoin() ? " LEFT JOIN "
- : isLeftOrInnerJoin() ? " LEFT OR INNER JOIN " : " INNER JOIN ")
- .append(right()).append(" ON ")
- .append(Arrays.toString(Arrays.stream(leftCols).map(ColumnDesc::getName).toArray())).append(" = ")
- .append(Arrays.toString(Arrays.stream(rightCols).map(ColumnDesc::getName).toArray()));
- return sb.toString();
- }
- }
-
- private Edge edgeOf(JoinDesc join) {
- return new Edge(join);
- }
-
- private static final IJoinEdgeMatcher DEFAULT_JOIN_EDGE_MATCHER = new DefaultJoinEdgeMatcher();
- @Setter
- private IJoinEdgeMatcher joinEdgeMatcher = DEFAULT_JOIN_EDGE_MATCHER;
-
- /**
- * compare:
- * 1. JoinType
- * 2. Columns on both sides
- */
- public interface IJoinEdgeMatcher extends Serializable {
- boolean matches(@NonNull Edge join1, @NonNull Edge join2);
- }
-
- public static class DefaultJoinEdgeMatcher implements IJoinEdgeMatcher {
- @Override
- public boolean matches(@NonNull Edge join1, @NonNull Edge join2) {
- if (join1.isLeftJoin() != join2.isLeftJoin() && !join1.isLeftOrInnerJoin() && !join2.isLeftOrInnerJoin()) {
- return false;
- }
-
- if (!Objects.equals(join1.nonEquiJoinCondition, join2.nonEquiJoinCondition)) {
- return false;
- }
-
- if (join1.isLeftJoin()) {
- return columnDescEquals(join1.leftCols, join2.leftCols)
- && columnDescEquals(join1.rightCols, join2.rightCols);
- } else {
- return (columnDescEquals(join1.leftCols, join2.leftCols)
- && columnDescEquals(join1.rightCols, join2.rightCols))
- || (columnDescEquals(join1.leftCols, join2.rightCols)
- && columnDescEquals(join1.rightCols, join2.leftCols));
- }
- }
-
- private boolean columnDescEquals(ColumnDesc[] a, ColumnDesc[] b) {
- if (a.length != b.length)
- return false;
-
- for (int i = 0; i < a.length; i++) {
- if (!columnDescEquals(a[i], b[i]))
- return false;
- }
- return true;
- }
-
- protected boolean columnDescEquals(ColumnDesc a, ColumnDesc b) {
- return Objects.equals(a, b);
- }
- }
-
- @Getter
- private final TableRef center;
- private final Map<String, TableRef> nodes = new HashMap<>();
- private final Set<Edge> edges = new HashSet<>();
- private final Map<TableRef, List<Edge>> edgesFromNode = new HashMap<>();
- private final Map<TableRef, List<Edge>> edgesToNode = new HashMap<>();
-
- /**
- * For model there's always a center, if there's only one tableScan it's the center.
- * Otherwise the center is not determined, it's a linked graph, hard to tell the center.
- */
- public JoinsGraph(TableRef root, List<JoinDesc> joins) {
- this.center = root;
- addNode(root);
-
- for (JoinDesc join : joins) {
- Preconditions.checkState(Arrays.stream(join.getForeignKeyColumns()).allMatch(TblColRef::isQualified));
- Preconditions.checkState(Arrays.stream(join.getPrimaryKeyColumns()).allMatch(TblColRef::isQualified));
- addAsEdge(join);
- }
-
- validate(joins);
- }
-
- private void addNode(TableRef table) {
- Preconditions.checkNotNull(table);
- String alias = table.getAlias();
- TableRef node = nodes.get(alias);
- if (node != null) {
- Preconditions.checkArgument(node.equals(table), "[%s]'s Alias \"%s\" has conflict with [%s].", table, alias,
- node);
- } else {
- nodes.put(alias, table);
- }
- }
-
- private void addAsEdge(JoinDesc join) {
- TableRef fkTable = join.getFKSide();
- TableRef pkTable = join.getPKSide();
- addNode(pkTable);
-
- Edge edge = edgeOf(join);
- edgesFromNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
- edgesFromNode.get(fkTable).add(edge);
- edgesToNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
- edgesToNode.get(pkTable).add(edge);
- if (!edge.isLeftJoin()) {
- // inner join is reversible
- edgesFromNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
- edgesFromNode.get(pkTable).add(edge);
- edgesToNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
- edgesToNode.get(fkTable).add(edge);
- }
- edges.add(edge);
- }
-
- public void setJoinToLeftOrInner(JoinDesc join) {
- if (!join.isLeftJoin()) {
- join.setLeftOrInner(true);
- return;
- }
-
- join.setLeftOrInner(true);
- TableRef fkTable = join.getFKSide();
- TableRef pkTable = join.getPKSide();
- Edge edge = edges.stream().filter(e -> e.isJoinMatched(join)).findFirst().orElse(null);
- if (edge == null) {
- return;
- }
- edgesFromNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
- edgesFromNode.get(pkTable).add(edge);
- edgesToNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
- edgesToNode.get(fkTable).add(edge);
- }
-
- private void validate(List<JoinDesc> joins) {
- for (JoinDesc join : joins) {
- TableRef fkTable = join.getFKSide();
- Preconditions.checkNotNull(nodes.get(fkTable.getAlias()));
- Preconditions.checkState(nodes.get(fkTable.getAlias()).equals(fkTable));
- }
- Preconditions.checkState(nodes.size() == joins.size() + 1);
- }
-
- public boolean match(JoinsGraph pattern, Map<String, String> matchAlias) {
- return match(pattern, matchAlias, false);
- }
-
- public boolean match(JoinsGraph pattern, Map<String, String> matchAlias, boolean matchPatial) {
- return match(pattern, matchAlias, matchPatial, false);
- }
-
- public boolean match(JoinsGraph pattern, Map<String, String> matchAlias, boolean matchPatial,
- boolean matchPartialNonEquiJoin) {
- if (pattern == null || pattern.center == null) {
- throw new IllegalArgumentException("pattern(model) should have a center: " + pattern);
- }
-
- List<TableRef> candidatesOfQCenter = searchCenterByIdentity(pattern.center);
- if (CollectionUtils.isEmpty(candidatesOfQCenter)) {
- return false;
- }
-
- for (TableRef queryCenter : candidatesOfQCenter) {
- // query <-> pattern
- Map<TableRef, TableRef> trialMatch = Maps.newHashMap();
- trialMatch.put(queryCenter, pattern.center);
-
- if (!checkInnerJoinNum(pattern, queryCenter, pattern.center, matchPatial)) {
- continue;
- }
-
- AtomicReference<Map<TableRef, TableRef>> finalMatchRef = new AtomicReference<>();
- innerMatch(pattern, trialMatch, matchPatial, finalMatchRef);
- if (finalMatchRef.get() != null
- && (matchPartialNonEquiJoin || checkNonEquiJoinMatches(finalMatchRef.get(), pattern))) {
- matchAlias.clear();
- matchAlias.putAll(finalMatchRef.get().entrySet().stream()
- .collect(Collectors.toMap(e -> e.getKey().getAlias(), e -> e.getValue().getAlias())));
- return true;
- }
- }
- return false;
- }
-
- public static JoinsGraph normalizeJoinGraph(JoinsGraph joinsGraph) {
- for (Edge edge : joinsGraph.edges) {
- if (!edge.isLeftJoin() || edge.isLeftOrInnerJoin()) {
- TableRef leftTable = edge.left();
- List<Edge> edgeList = joinsGraph.edgesToNode.get(leftTable);
- if (CollectionUtils.isEmpty(edgeList)) {
- continue;
- }
- for (Edge targetEdge : edgeList) {
- if (!edge.equals(targetEdge) && leftTable.equals(targetEdge.right())
- && !targetEdge.isLeftOrInnerJoin()) {
- joinsGraph.setJoinToLeftOrInner(targetEdge.join);
- normalizeJoinGraph(joinsGraph);
- }
- }
- }
- }
- return joinsGraph;
- }
-
- public List<TableRef> getAllTblRefNodes() {
- return Lists.newArrayList(nodes.values());
- }
-
- /**
- * check if any non-equi join is missed in the pattern
- * if so, we cannot match the current graph with the the pattern graph.
- * set `kylin.query.match-partial-non-equi-join-model` to skip this checking
- * @param matches
- * @return
- */
- private boolean checkNonEquiJoinMatches(Map<TableRef, TableRef> matches, JoinsGraph pattern) {
- HashSet<TableRef> patternGraphTables = new HashSet<>(pattern.nodes.values());
-
- for (TableRef patternTable : patternGraphTables) {
- List<Edge> outgoingEdges = pattern.getEdgesByFKSide(patternTable);
- // for all outgoing non-equi join edges
- // if there is no match found for the right side table in the current graph
- // return false
- for (Edge outgoingEdge : outgoingEdges) {
- if (outgoingEdge.isNonEquiJoin()) {
- if (!matches.containsValue(patternTable) || !matches.containsValue(outgoingEdge.right())) {
- return false;
- }
- }
- }
- }
- return true;
- }
-
- private boolean isAllJoinInner(JoinsGraph joinsGraph, TableRef tableRef) {
- List<Edge> edgesFromNode = joinsGraph.edgesFromNode.get(tableRef);
- List<Edge> edgesToNode = joinsGraph.edgesToNode.get(tableRef);
-
- if (edgesFromNode == null) {
- return false;
- }
-
- if (edgesToNode == null) {
- return false;
- }
-
- if (edgesToNode.size() != edgesFromNode.size()) {
- return false;
- }
-
- for (Edge edge : edgesFromNode) {
- if (edge.join.isLeftJoin()) {
- return false;
- }
- }
-
- return true;
- }
-
- private boolean checkInnerJoinNum(JoinsGraph pattern, TableRef queryTableRef, TableRef patternTableRef,
- boolean matchPartial) {
- if (matchPartial) {
- return true;
- }
- // fully match: unmatched if extra inner join edge on either graph
- // matched if: query graph join count: model graph join count:
- // 1) inner join <= inner join
- // 2) inner join + left or inner join >= inner join
- List<Edge> innerQueryEdges = this.edgesFrom(queryTableRef).stream().filter(Edge::isInnerJoin)
- .collect(Collectors.toList());
- List<Edge> notLeftQueryEdges = this.edgesFrom(queryTableRef).stream().filter(e -> !e.isLeftJoin())
- .collect(Collectors.toList());
- List<Edge> innerPatternEdges = pattern.edgesFrom(patternTableRef).stream().filter(Edge::isInnerJoin)
- .collect(Collectors.toList());
-
- // if all joins are inner joins, compare sum of both sides
- if (isAllJoinInner(this, queryTableRef) && isAllJoinInner(pattern, patternTableRef)) {
- int cntInnerQueryEdges = innerQueryEdges.size();
- int cntNotLeftQueryEdges = notLeftQueryEdges.size();
- int cntInnerPatternEdges = innerPatternEdges.size();
- return cntInnerQueryEdges <= cntInnerPatternEdges && cntNotLeftQueryEdges >= cntInnerPatternEdges;
- }
-
- // if not all joins are inner, compare left side and right side separately
- // Calculate join count in query graph
- int cntLeftSideInnerQueryEdges = (int) innerQueryEdges.stream()
- .filter(edge -> edge.right().equals(queryTableRef)).count();
- int cntRightSideInnerQueryEdges = (int) innerQueryEdges.stream()
- .filter(edge -> edge.left().equals(queryTableRef)).count();
- int cntLeftSideNotLeftQueryEdges = (int) notLeftQueryEdges.stream()
- .filter(edge -> edge.right().equals(queryTableRef)).count();
- int cntRightSideNotLeftQueryEdges = (int) notLeftQueryEdges.stream()
- .filter(edge -> edge.left().equals(queryTableRef)).count();
- // Calculate join count in model graph
- int cntLeftSideInnerPatternEdges = (int) innerPatternEdges.stream()
- .filter(edge -> edge.right().equals(patternTableRef)).count();
- int cntRightSideInnerPatternEdges = (int) innerPatternEdges.stream()
- .filter(edge -> edge.left().equals(patternTableRef)).count();
-
- boolean isLeftEqual = cntLeftSideInnerQueryEdges <= cntLeftSideInnerPatternEdges
- && cntLeftSideNotLeftQueryEdges >= cntLeftSideInnerPatternEdges;
- boolean isRightEqual = cntRightSideInnerQueryEdges <= cntRightSideInnerPatternEdges
- && cntRightSideNotLeftQueryEdges >= cntRightSideInnerPatternEdges;
- return isLeftEqual && isRightEqual;
- }
-
- private void innerMatch(JoinsGraph pattern, Map<TableRef, TableRef> trialMatches, boolean matchPartial,
- AtomicReference<Map<TableRef, TableRef>> finalMatch) {
- if (trialMatches.size() == nodes.size()) {
- //match is found
- finalMatch.set(trialMatches);
- return;
- }
-
- Preconditions.checkState(nodes.size() > trialMatches.size());
- Optional<Pair<Edge, TableRef>> toMatch = trialMatches.keySet().stream()
- .map(t -> edgesFrom(t).stream().filter(e -> !trialMatches.containsKey(e.other(t))).findFirst()
- .map(edge -> new Pair<>(edge, edge.other(t))).orElse(null))
- .filter(Objects::nonNull).findFirst();
-
- Preconditions.checkState(toMatch.isPresent());
- Edge toMatchQueryEdge = toMatch.get().getFirst();
- TableRef toMatchQueryNode = toMatch.get().getSecond();
- TableRef matchedQueryNode = toMatchQueryEdge.other(toMatchQueryNode);
- TableRef matchedPatternNode = trialMatches.get(matchedQueryNode);
-
- List<TableRef> toMatchPatternNodeCandidates = Lists.newArrayList();
- for (Edge patternEdge : pattern.edgesFrom(matchedPatternNode)) {
- TableRef toMatchPatternNode = patternEdge.other(matchedPatternNode);
- if (!toMatchQueryNode.getTableIdentity().equals(toMatchPatternNode.getTableIdentity())
- || !toMatchQueryEdge.equals(patternEdge) || trialMatches.containsValue(toMatchPatternNode)
- || !checkInnerJoinNum(pattern, toMatchQueryNode, toMatchPatternNode, matchPartial)) {
- continue;
- }
- toMatchPatternNodeCandidates.add(toMatchPatternNode);
- }
-
- for (TableRef toMatchPatternNode : toMatchPatternNodeCandidates) {
- Map<TableRef, TableRef> newTrialMatches = Maps.newHashMap();
- newTrialMatches.putAll(trialMatches);
- newTrialMatches.put(toMatchQueryNode, toMatchPatternNode);
- innerMatch(pattern, newTrialMatches, matchPartial, finalMatch);
- if (finalMatch.get() != null) {
- //get out of recursive invoke chain straightly
- return;
- }
- }
- }
-
- public List<Edge> unmatched(JoinsGraph pattern) {
- List<Edge> unmatched = Lists.newArrayList();
- Set<Edge> all = edgesFromNode.values().stream().flatMap(List::stream).collect(Collectors.toSet());
- for (Edge edge : all) {
- List<JoinDesc> joins = getJoinsPathByPKSide(edge.right());
- JoinsGraph subGraph = new JoinsGraph(center, joins);
- if (subGraph.match(pattern, Maps.newHashMap())) {
- continue;
- }
- unmatched.add(edge);
- }
- return unmatched;
- }
-
- private List<TableRef> searchCenterByIdentity(final TableRef table) {
- // special case: several same nodes in a JoinGraph
- return nodes.values().stream().filter(node -> node.getTableIdentity().equals(table.getTableIdentity()))
- .filter(node -> {
- List<JoinDesc> path2Center = getJoinsPathByPKSide(node);
- return path2Center.stream().noneMatch(JoinDesc::isLeftJoin);
- }).collect(Collectors.toList());
- }
-
- private List<Edge> edgesFrom(TableRef thisSide) {
- return edgesFromNode.getOrDefault(thisSide, Lists.newArrayList());
- }
-
- public Map<String, String> matchAlias(JoinsGraph joinsGraph, KylinConfig kylinConfig) {
- Map<String, String> matchAlias = Maps.newHashMap();
- match(joinsGraph, matchAlias, kylinConfig.isQueryMatchPartialInnerJoinModel(),
- kylinConfig.partialMatchNonEquiJoins());
- return matchAlias;
- }
-
- public Map<String, String> matchAlias(JoinsGraph joinsGraph, boolean matchPartial) {
- Map<String, String> matchAlias = Maps.newHashMap();
- match(joinsGraph, matchAlias, matchPartial);
- return matchAlias;
- }
-
- public List<Edge> getEdgesByFKSide(TableRef table) {
- if (!edgesFromNode.containsKey(table)) {
- return Lists.newArrayList();
- }
- return edgesFromNode.get(table).stream().filter(e -> e.isFkSide(table)).collect(Collectors.toList());
- }
-
- private Edge getEdgeByPKSide(TableRef table) {
- if (!edgesToNode.containsKey(table)) {
- return null;
- }
- List<Edge> edgesByPkSide = edgesToNode.get(table).stream().filter(e -> e.isPkSide(table))
- .collect(Collectors.toList());
- if (edgesByPkSide.isEmpty()) {
- return null;
- }
- Preconditions.checkState(edgesByPkSide.size() == 1, "%s is allowed to be Join PK side once", table);
- return edgesByPkSide.get(0);
- }
-
- public JoinDesc getJoinByPKSide(TableRef table) {
- Edge edge = getEdgeByPKSide(table);
- return edge != null ? edge.join : null;
- }
-
- private List<JoinDesc> getJoinsPathByPKSide(TableRef table) {
- List<JoinDesc> pathToRoot = Lists.newArrayList();
- TableRef pkSide = table; // start from leaf
- while (pkSide != null) {
- JoinDesc subJoin = getJoinByPKSide(pkSide);
- if (subJoin != null) {
- pathToRoot.add(subJoin);
- pkSide = subJoin.getFKSide();
- } else {
- pkSide = null;
- }
- }
- return Lists.reverse(pathToRoot);
- }
-
- public JoinsGraph getSubgraphByAlias(Set<String> aliasSets) {
- TableRef subGraphRoot = this.center;
- Set<JoinDesc> subGraphJoin = Sets.newHashSet();
- for (String alias : aliasSets) {
- subGraphJoin.addAll(getJoinsPathByPKSide(nodes.get(alias)));
- }
- return new JoinsGraph(subGraphRoot, Lists.newArrayList(subGraphJoin));
- }
-
- @Override
- public String toString() {
- StringBuilder graphStrBuilder = new StringBuilder();
- graphStrBuilder.append("Root: ").append(center);
- List<Edge> nextEdges = getEdgesByFKSide(center);
- nextEdges.forEach(e -> buildGraphStr(graphStrBuilder, e, 1));
- return graphStrBuilder.toString();
- }
-
- private void buildGraphStr(StringBuilder sb, @NonNull Edge edge, int indent) {
- sb.append('\n');
- for (int i = 0; i < indent; i++) {
- sb.append(" ");
- }
- sb.append(edge);
- List<Edge> nextEdges = getEdgesByFKSide(edge.right());
- nextEdges.forEach(e -> buildGraphStr(sb, e, indent + 1));
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 9c6209ec59..3d7cffbd17 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -55,6 +55,7 @@ import org.apache.kylin.common.scheduler.SchedulerEventNotifier;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.project.NProjectManager;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/DefaultJoinEdgeMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/DefaultJoinEdgeMatcher.java
new file mode 100644
index 0000000000..a1e0402e45
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/DefaultJoinEdgeMatcher.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model.graph;
+
+import java.util.List;
+import java.util.Objects;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.metadata.model.ColumnDesc;
+
+import lombok.NonNull;
+
+public class DefaultJoinEdgeMatcher implements IJoinEdgeMatcher {
+
+ @Override
+ public boolean matches(@NonNull Edge join1, @NonNull Edge join2) {
+ if (join1.isLeftJoin() != join2.isLeftJoin() && !join1.isLeftOrInnerJoin() && !join2.isLeftOrInnerJoin()) {
+ return false;
+ }
+
+ if (!Objects.equals(join1.nonEquiJoinCondition, join2.nonEquiJoinCondition)) {
+ return false;
+ }
+
+ if (join1.isLeftJoin()) {
+ return columnDescEquals(join1.leftCols, join2.leftCols)
+ && columnDescEquals(join1.rightCols, join2.rightCols);
+ } else {
+ return (columnDescEquals(join1.leftCols, join2.leftCols)
+ && columnDescEquals(join1.rightCols, join2.rightCols))
+ || (columnDescEquals(join1.leftCols, join2.rightCols)
+ && columnDescEquals(join1.rightCols, join2.leftCols));
+ }
+ }
+
+ private boolean columnDescEquals(ColumnDesc[] a, ColumnDesc[] b) {
+ if (a.length != b.length) {
+ return false;
+ }
+
+ List<ColumnDesc> oneList = Lists.newArrayList(a);
+ List<ColumnDesc> anotherList = Lists.newArrayList(b);
+
+ for (ColumnDesc obj : oneList) {
+ anotherList.removeIf(dual -> columnDescEquals(obj, dual));
+ }
+ return anotherList.isEmpty();
+ }
+
+ protected boolean columnDescEquals(ColumnDesc a, ColumnDesc b) {
+ return Objects.equals(a, b);
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/Edge.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/Edge.java
new file mode 100644
index 0000000000..58fcdafe51
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/Edge.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model.graph;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Objects;
+
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.NonEquiJoinCondition;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import lombok.Getter;
+import lombok.Setter;
+
+public class Edge implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public final JoinDesc join;
+ public final ColumnDesc[] leftCols;
+ public final ColumnDesc[] rightCols;
+ public final NonEquiJoinCondition nonEquiJoinCondition;
+ @Setter
+ private IJoinEdgeMatcher joinEdgeMatcher = new DefaultJoinEdgeMatcher();
+ @Setter
+ @Getter
+ private boolean swapJoin;
+
+ public Edge(JoinDesc join) {
+ this(join, false);
+ }
+
+ public Edge(JoinDesc join, boolean swapJoin) {
+ this.join = join;
+
+ int i = 0;
+ leftCols = new ColumnDesc[join.getForeignKeyColumns().length];
+ for (TblColRef colRef : join.getForeignKeyColumns()) {
+ leftCols[i++] = colRef.getColumnDesc();
+ }
+
+ i = 0;
+ rightCols = new ColumnDesc[join.getPrimaryKeyColumns().length];
+ for (TblColRef colRef : join.getPrimaryKeyColumns()) {
+ rightCols[i++] = colRef.getColumnDesc();
+ }
+
+ nonEquiJoinCondition = join.getNonEquiJoinCondition();
+ this.swapJoin = swapJoin;
+ }
+
+ public boolean isJoinMatched(JoinDesc other) {
+ return join.equals(other);
+ }
+
+ public boolean isNonEquiJoin() {
+ return nonEquiJoinCondition != null;
+ }
+
+ public boolean isLeftJoin() {
+ return !join.isLeftOrInnerJoin() && join.isLeftJoin();
+ }
+
+ public boolean isLeftOrInnerJoin() {
+ return join.isLeftOrInnerJoin();
+ }
+
+ public TableRef pkSide() {
+ return join.getPKSide();
+ }
+
+ public TableRef fkSide() {
+ return join.getFKSide();
+ }
+
+ public boolean isFkSide(TableRef tableRef) {
+ return fkSide().equals(tableRef);
+ }
+
+ public boolean isPkSide(TableRef tableRef) {
+ return pkSide().equals(tableRef);
+ }
+
+ public TableRef otherSide(TableRef tableRef) {
+ if (isFkSide(tableRef)) {
+ return pkSide();
+ } else if (isPkSide(tableRef)) {
+ return fkSide();
+ }
+ throw new IllegalArgumentException("table " + tableRef + " is not on the edge: " + this);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+
+ if (this.getClass() != other.getClass()) {
+ return false;
+ }
+ return joinEdgeMatcher.matches(this, (Edge) other);
+ }
+
+ @Override
+ public int hashCode() {
+ if (this.isLeftJoin()) {
+ return Objects.hash(isLeftJoin(), leftCols, rightCols);
+ }
+ if (Arrays.hashCode(leftCols) < Arrays.hashCode(rightCols)) {
+ return Objects.hash(isLeftJoin(), leftCols, rightCols);
+ }
+ return Objects.hash(isLeftJoin(), rightCols, leftCols);
+ }
+
+ @Override
+ public String toString() {
+ // Edge: TableRef[TEST_KYLIN_FACT] LEFT JOIN TableRef[TEST_ORDER] ON [ORDER_ID] = [ORDER_ID]
+ return "Edge: " + join.getFKSide() + getJoinTypeStr() + join.getPKSide() + " ON "
+ + Arrays.toString(Arrays.stream(leftCols).map(ColumnDesc::getName).toArray()) + " = "
+ + Arrays.toString(Arrays.stream(rightCols).map(ColumnDesc::getName).toArray());
+ }
+
+ private String getJoinTypeStr() {
+ if (isLeftJoin()) {
+ return " LEFT JOIN ";
+ }
+ return isLeftOrInnerJoin() ? " LEFT OR INNER JOIN " : " INNER JOIN ";
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/IJoinEdgeMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/IJoinEdgeMatcher.java
new file mode 100644
index 0000000000..be5376b90a
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/IJoinEdgeMatcher.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model.graph;
+
+import java.io.Serializable;
+
+import lombok.NonNull;
+
+public interface IJoinEdgeMatcher extends Serializable {
+ boolean matches(@NonNull Edge join1, @NonNull Edge join2);
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
new file mode 100644
index 0000000000..b6342fd476
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
@@ -0,0 +1,506 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model.graph;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.val;
+
+public class JoinsGraph implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Getter
+ private final TableRef center;
+ @Getter
+ private final Map<String, TableRef> vertexMap = Maps.newLinkedHashMap();
+ private final Map<TableRef, VertexInfo<Edge>> vertexInfoMap = Maps.newHashMap();
+ private final Set<Edge> edges = Sets.newHashSet();
+
+ /**
+ * Creates a graph
+ */
+ public JoinsGraph(TableRef root, List<JoinDesc> joins) {
+ this(root, joins, true);
+ }
+
+ public JoinsGraph(TableRef root, List<JoinDesc> joins, boolean needSwapJoin) {
+ this.center = root;
+ addVertex(root);
+
+ List<Pair<JoinDesc, Boolean>> newJoinsPair = swapJoinDescs(joins, needSwapJoin);
+ for (Pair<JoinDesc, Boolean> pair : newJoinsPair) {
+ JoinDesc join = pair.getFirst();
+ Boolean isSwap = pair.getSecond();
+
+ Preconditions.checkState(Arrays.stream(join.getForeignKeyColumns()).allMatch(TblColRef::isQualified));
+ Preconditions.checkState(Arrays.stream(join.getPrimaryKeyColumns()).allMatch(TblColRef::isQualified));
+ addVertex(join.getPKSide());
+ addVertex(join.getFKSide());
+ addEdge(join, isSwap);
+ }
+ validate(joins);
+ }
+
+ static class VertexInfo<E> {
+ final List<E> outEdges = new ArrayList<>();
+ final List<E> inEdges = new ArrayList<>();
+ }
+
+ public void addVertex(TableRef table) {
+ if (vertexMap.containsKey(table.getAlias())) {
+ return;
+ }
+ vertexMap.put(table.getAlias(), table);
+ vertexInfoMap.computeIfAbsent(table, f -> new VertexInfo<>());
+ }
+
+ public void addEdge(JoinDesc join, boolean swapJoin) {
+ Edge edge = new Edge(join, swapJoin);
+ vertexInfoMap.get(join.getPKSide()).inEdges.add(edge);
+ vertexInfoMap.get(join.getFKSide()).outEdges.add(edge);
+ edges.add(edge);
+ }
+
+ private void validate(List<JoinDesc> joins) {
+ for (JoinDesc join : joins) {
+ TableRef fkSide = join.getFKSide();
+ Preconditions.checkNotNull(vertexMap.get(fkSide.getAlias()));
+ Preconditions.checkState(vertexMap.get(fkSide.getAlias()).equals(fkSide));
+ }
+ Preconditions.checkState(vertexMap.size() == joins.size() + 1);
+ }
+
+ private List<Pair<JoinDesc, Boolean>> swapJoinDescs(List<JoinDesc> joins, boolean needSwapJoin) {
+ List<Pair<JoinDesc, Boolean>> newJoins = Lists.newArrayList();
+ for (JoinDesc join : joins) {
+ // add origin joinDesc
+ newJoins.add(Pair.newPair(join, false));
+ // inner / leftOrInner => swap joinDesc
+ if ((join.isInnerJoin() || join.isLeftOrInnerJoin()) && needSwapJoin) {
+ newJoins.add(Pair.newPair(swapJoinDesc(join), true));
+ }
+ }
+ return newJoins;
+ }
+
+ private JoinDesc swapJoinDesc(JoinDesc originJoinDesc) {
+ JoinDesc swapedJoinDesc = new JoinDesc();
+ swapedJoinDesc.setType(originJoinDesc.getType());
+ swapedJoinDesc.setPrimaryKey(originJoinDesc.getForeignKey());
+ swapedJoinDesc.setForeignKey(originJoinDesc.getPrimaryKey());
+ swapedJoinDesc.setNonEquiJoinCondition(originJoinDesc.getNonEquiJoinCondition());
+ swapedJoinDesc.setPrimaryTable(originJoinDesc.getForeignTable());
+ swapedJoinDesc.setForeignTable(originJoinDesc.getPrimaryTable());
+ swapedJoinDesc.setPrimaryKeyColumns(originJoinDesc.getForeignKeyColumns());
+ swapedJoinDesc.setForeignKeyColumns(originJoinDesc.getPrimaryKeyColumns());
+ swapedJoinDesc.setPrimaryTableRef(originJoinDesc.getForeignTableRef());
+ swapedJoinDesc.setForeignTableRef(originJoinDesc.getPrimaryTableRef());
+ // swap left join, the current join must be convertible to LeftOrInner
+ swapedJoinDesc.setLeftOrInner(originJoinDesc.isLeftOrInnerJoin());
+ return swapedJoinDesc;
+ }
+
+ public void setJoinEdgeMatcher(IJoinEdgeMatcher joinEdgeMatcher) {
+ edges.forEach(edge -> edge.setJoinEdgeMatcher(joinEdgeMatcher));
+ }
+
+ public List<Edge> outwardEdges(TableRef table) {
+ return Lists.newArrayList(vertexInfoMap.get(table).outEdges);
+ }
+
+ public List<Edge> inwardEdges(TableRef table) {
+ return Lists.newArrayList(vertexInfoMap.get(table).inEdges);
+ }
+
+ public String getCenterTableIdentity() {
+ return center.getTableIdentity();
+ }
+
+ public boolean match(JoinsGraph pattern, Map<String, String> matchAliasMap) {
+ return match(pattern, matchAliasMap, false);
+ }
+
+ public boolean match(JoinsGraph pattern, Map<String, String> matchAliasMap, boolean matchPartial) {
+ return match(pattern, matchAliasMap, matchPartial, false);
+ }
+
+ public boolean match(JoinsGraph pattern, Map<String, String> matchAliasMap, boolean matchPartial,
+ boolean matchPartialNonEquiJoin) {
+ if (Objects.isNull(pattern) || Objects.isNull(pattern.center)) {
+ throw new IllegalArgumentException("pattern(model) should have a center: " + pattern);
+ }
+
+ List<TableRef> candidatesOfQCenter = searchCenter(pattern.getCenterTableIdentity());
+ for (TableRef candidateCenter : candidatesOfQCenter) {
+ List<Edge> unmatchedPatternOutEdges = Lists.newArrayList();
+ Map<TableRef, TableRef> allMatchedTableMap = Maps.newHashMap();
+ List<Pair<TableRef, TableRef>> toMatchTableList = Lists.newArrayList();
+ toMatchTableList.add(Pair.newPair(candidateCenter, pattern.center));
+
+ match0(pattern, toMatchTableList, unmatchedPatternOutEdges, allMatchedTableMap);
+
+ if (!toMatchTableList.isEmpty() || allMatchedTableMap.size() != this.vertexMap.size()) {
+ continue;
+ }
+
+ // There are three cases of match, the first two cases are exactly match.
+ // 1. pattern out edges is matched.
+ // 2. unmatched out edges of pattern is not empty, but all of them are left join.
+ // 3. unmatched out edges of pattern is not empty, but match partial.
+ if ((unmatchedPatternOutEdges.isEmpty() || unmatchedPatternOutEdges.stream().allMatch(Edge::isLeftJoin)
+ || matchPartial) && !allMatchedTableMap.isEmpty()
+ && (matchPartialNonEquiJoin || checkNonEquiJoinMatches(allMatchedTableMap, pattern))) {
+ matchAliasMap.clear();
+ matchAliasMap.putAll(allMatchedTableMap.entrySet().stream()
+ .collect(Collectors.toMap(e -> e.getKey().getAlias(), e -> e.getValue().getAlias())));
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Match this joinGraph with a pattern's joinGraph. All matched tables will be put
+ * into the parameter {@code matchedTableMap}. The pattern's unmatched out edges
+ * will be put into the parameter {@code unmatchedOutEdgesOfP}.
+ *
+ * @param pattern the joinGraph to compare, usually the model's joinGraph.
+ * @param toMatchTableList tables to match, the keys from this graph, the values from the joinGraph of pattern.
+ * @param unmatchedPatternOutEdges unmatched out edges of the pattern.
+ * @param matchedTableMap All matched tables come from the toMatchMap.
+ */
+ private void match0(JoinsGraph pattern, List<Pair<TableRef, TableRef>> toMatchTableList,
+ List<Edge> unmatchedPatternOutEdges, Map<TableRef, TableRef> matchedTableMap) {
+
+ // The toMatchTableList will add nodes to be matched in the loop.
+ // Traverse toMatchTableList to match each table
+ for (int i = 0; i < toMatchTableList.size(); i++) {
+
+ Pair<TableRef, TableRef> pair = toMatchTableList.get(i);
+ TableRef queryFKSide = pair.getFirst();
+ TableRef patternFKSide = pair.getSecond();
+ List<Edge> queryOutEdges = this.outwardEdges(queryFKSide);
+ Set<Edge> patternOutEdges = Sets.newHashSet(pattern.outwardEdges(patternFKSide));
+
+ // Traverse queryOutEdgesIter to match each edge from patternOutEdges
+ // Edges that match successfully will be deleted from queryOutEdges and patternOutEdges
+ // When all outgoing edges of the query table can be matched in the outgoing edges of the pattern table
+ Iterator<Edge> queryOutEdgesIter = queryOutEdges.iterator();
+ while (queryOutEdgesIter.hasNext()) {
+
+ Edge queryOutEdge = queryOutEdgesIter.next();
+ TableRef queryPKSide = queryOutEdge.otherSide(queryFKSide);
+ Edge matchedPatternEdge = findOutEdgeFromDualTable(pattern, patternOutEdges, queryPKSide, queryOutEdge);
+ boolean patternEdgeNotMatch = Objects.isNull(matchedPatternEdge);
+
+ // query: A leftOrInner B (left or inner -> leftOrInner)
+ // pattern: A left B
+ // edgeBA is not found in patternOutEdges, but this case can be matched
+ if (queryOutEdge.isLeftOrInnerJoin() && (patternOutEdges.isEmpty() || patternEdgeNotMatch)) {
+ queryOutEdgesIter.remove();
+ } else {
+ // can't find the same edge
+ if (patternEdgeNotMatch) {
+ break;
+ }
+ queryOutEdgesIter.remove();
+ patternOutEdges.remove(matchedPatternEdge);
+ // both out edge pk side tables add to toMatchTableList for the next round of the loop
+ TableRef patternPKSide = matchedPatternEdge.otherSide(patternFKSide);
+ addIfAbsent(toMatchTableList, Pair.newPair(queryPKSide, patternPKSide));
+ }
+ }
+
+ // All queryOutEdges are not matched in patternOutEdges
+ if (CollectionUtils.isNotEmpty(queryOutEdges)) {
+ break;
+ }
+ // All queryOutEdges are successfully matched in patternOutEdges.
+ // put queryFKSide and patternFKSide to matchedTableMap
+ matchedTableMap.put(queryFKSide, patternFKSide);
+ // The remaining edges in patternOutEdges are all put into unmatchedPatternOutEdges
+ unmatchedPatternOutEdges.addAll(patternOutEdges);
+ }
+ // intersect the table to be matched with the table already matched
+ val matchedList = matchedTableMap.entrySet().stream().map(e -> Pair.newPair(e.getKey(), e.getValue()))
+ .collect(Collectors.toList());
+ toMatchTableList.removeAll(matchedList);
+ }
+
+ /**
+ * Find the out edge from the dual {@code TableRef}.
+ *
+ * @return the dual table's out edge in joinGraph of pattern.
+ */
+ private Edge findOutEdgeFromDualTable(JoinsGraph pattern, Set<Edge> patternOutEdges, TableRef queryPKSide,
+ Edge queryOutEdge) {
+ Set<Edge> matchedEdges = patternOutEdges.stream()
+ .filter(outPatternEdge -> StringUtils.equals(queryPKSide.getTableIdentity(),
+ outPatternEdge.pkSide().getTableIdentity()) && queryOutEdge.equals(outPatternEdge))
+ .collect(Collectors.toSet());
+ if (matchedEdges.size() == 1) {
+ return matchedEdges.iterator().next();
+ }
+ for (Edge matchedEdge : matchedEdges) {
+ TableRef patternPKSide = matchedEdge.pkSide();
+ int queryOutEdgeSize = this.vertexInfoMap.get(queryPKSide).outEdges.size();
+ int patternOutEdgeSize = pattern.vertexInfoMap.get(patternPKSide).outEdges.size();
+ if (queryOutEdgeSize != patternOutEdgeSize) {
+ continue;
+ }
+ return matchedEdge;
+ }
+ return null;
+ }
+
+ /**
+ * Sometimes one table may be joined more than one time.
+ *
+ * @param tableIdentity table identity(db.table_name) to search
+ * @return desired references of the input table identity
+ */
+ private List<TableRef> searchCenter(String tableIdentity) {
+ return vertexInfoMap.keySet().stream()
+ .filter(table -> StringUtils.equals(table.getTableIdentity(), tableIdentity))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Check if any non-equi join is missed in the pattern
+ * if so, we cannot match the current graph with the pattern graph.
+ * set `kylin.query.match-partial-non-equi-join-model` to skip this checking
+ */
+ public boolean checkNonEquiJoinMatches(Map<TableRef, TableRef> matches, JoinsGraph pattern) {
+ for (Map.Entry<TableRef, VertexInfo<Edge>> entry : pattern.vertexInfoMap.entrySet()) {
+ TableRef table = entry.getKey();
+ List<Edge> outEdges = entry.getValue().outEdges;
+ // for all outgoing non-equi join edges
+ // if there is no match found for the right side table in the current graph
+ // return false
+ for (Edge outgoingEdge : outEdges) {
+ if (outgoingEdge.isNonEquiJoin()
+ && (!matches.containsValue(table) || !matches.containsValue(outgoingEdge.pkSide()))) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ public List<TableRef> getAllTblRefNodes() {
+ return Lists.newArrayList(vertexMap.values());
+ }
+
+ /**
+ * Normalize joinsGraph. <p>
+ * 1. Find a path to the Inner or LeftOrInner edge. <p>
+ * 2. Recursively change all Left edges on the path to LeftOrInner edges.
+ * <p>
+ * Example:<p>
+ * query1: A Left B, B Left C, C Inner D => A LeftOrInner B, B LeftOrInner C, C Inner D <p>
+ * query2: A Left B, B Left C, C LeftOrInner D => A LeftOrInner B, B LeftOrInner C, C LeftOrInner D
+ * <p>
+ * If the PK table of Left Join has a non-null filter condition, then this Left Join has the same semantics as Inner Join <p>
+ * query: A Left B ON A.a = B.b Where B.c1 IS NOT NULL => A LeftOrInner B
+ */
+ public void normalize() {
+ Set<Edge> edgeSet = edges.stream().filter(e -> !e.isSwapJoin()).collect(Collectors.toSet());
+ for (Edge edge : edgeSet) {
+ if (!edge.isLeftJoin() || edge.isLeftOrInnerJoin()) {
+ TableRef fkSide = edge.fkSide();
+ List<Edge> edgeList = inwardEdges(fkSide);
+ if (CollectionUtils.isEmpty(edgeList)) {
+ continue;
+ }
+ for (Edge targetEdge : edgeList) {
+ if (!edge.equals(targetEdge) && fkSide.equals(targetEdge.pkSide())
+ && !targetEdge.isLeftOrInnerJoin() && targetEdge.isLeftJoin()) {
+ setJoinToLeftOrInner(targetEdge.join);
+ normalize();
+ }
+ }
+ }
+ }
+ }
+
+ public void setJoinToLeftOrInner(JoinDesc join) {
+ if (!join.isLeftJoin()) {
+ join.setLeftOrInner(true);
+ // inner -> leftOrInner, need swap another join
+ Edge swapEdge = edges.stream().filter(e -> e.isJoinMatched(swapJoinDesc(join))).findFirst().orElse(null);
+ if (swapEdge == null) {
+ return;
+ }
+ swapEdge.join.setLeftOrInner(true);
+ return;
+ }
+
+ Edge edge = edges.stream().filter(e -> e.isJoinMatched(join)).findFirst().orElse(null);
+ if (edge == null) {
+ return;
+ }
+
+ // change current join from left join to leftOrInner join
+ join.setLeftOrInner(true);
+ // left -> leftOrInner need swap join to create new edge
+ JoinDesc swapJoin = swapJoinDesc(join);
+ Edge swapEdge = new Edge(swapJoin, true);
+ vertexInfoMap.computeIfAbsent(swapEdge.pkSide(), f -> new VertexInfo<>());
+ vertexInfoMap.computeIfAbsent(swapEdge.fkSide(), f -> new VertexInfo<>());
+ addIfAbsent(vertexInfoMap.get(swapEdge.fkSide()).outEdges, swapEdge);
+ addIfAbsent(vertexInfoMap.get(swapEdge.pkSide()).inEdges, swapEdge);
+ if (edges.stream().noneMatch(e -> e.isJoinMatched(swapJoin))) {
+ edges.add(swapEdge);
+ }
+ }
+
+ public Map<String, String> matchAlias(JoinsGraph joinsGraph, KylinConfig kylinConfig) {
+ Map<String, String> matchAliasMap = Maps.newHashMap();
+ match(joinsGraph, matchAliasMap, kylinConfig.isQueryMatchPartialInnerJoinModel(),
+ kylinConfig.partialMatchNonEquiJoins());
+ return matchAliasMap;
+ }
+
+ public Map<String, String> matchAlias(JoinsGraph joinsGraph, boolean matchPartial) {
+ Map<String, String> matchAliasMap = Maps.newHashMap();
+ match(joinsGraph, matchAliasMap, matchPartial);
+ return matchAliasMap;
+ }
+
+ public JoinDesc getJoinByPKSide(TableRef pkTable) {
+ Edge edge = getEdgeByPKSide(pkTable);
+ return Objects.nonNull(edge) ? edge.join : null;
+ }
+
+ public List<Edge> getEdgesByFKSide(TableRef table) {
+ if (!vertexInfoMap.containsKey(table)) {
+ return Collections.emptyList();
+ }
+ return outwardEdges(table);
+ }
+
+ /**
+ * Get edges by primary key side at most get one edge.
+ *
+ * @param pkTable the pkSide table
+ * @return the obtained edge
+ */
+ private Edge getEdgeByPKSide(TableRef pkTable) {
+ if (!vertexInfoMap.containsKey(pkTable)) {
+ return null;
+ }
+ List<Edge> inEdges = inwardEdges(pkTable).stream().filter(edge -> !edge.isSwapJoin())
+ .collect(Collectors.toList());
+ if (inEdges.size() != 1) {
+ return null;
+ }
+ return inEdges.get(0);
+ }
+
+ public List<JoinDesc> getJoinsPathByPKSide(TableRef table) {
+ List<JoinDesc> pathToRoot = Lists.newArrayList();
+ TableRef pkSide = table;
+ while (pkSide != null) {
+ JoinDesc subJoin = getJoinByPKSide(pkSide);
+ if (Objects.isNull(subJoin)) {
+ pkSide = null;
+ } else {
+ pathToRoot.add(subJoin);
+ pkSide = subJoin.getFKSide();
+ }
+ }
+ return Lists.reverse(pathToRoot);
+ }
+
+ public JoinsGraph getSubGraphByAlias(Set<String> aliasSets) {
+ Set<JoinDesc> subJoins = Sets.newHashSet();
+ for (String alias : aliasSets) {
+ TableRef table = vertexMap.get(alias);
+ subJoins.addAll(getJoinsPathByPKSide(table));
+ }
+ return new JoinsGraph(this.center, Lists.newArrayList(subJoins));
+ }
+
+ public List<Edge> unmatched(JoinsGraph pattern) {
+ List<Edge> unmatched = Lists.newArrayList();
+ Set<Edge> all = vertexInfoMap.values().stream().map(m -> m.outEdges).flatMap(List::stream)
+ .collect(Collectors.toSet());
+ for (Edge edge : all) {
+ JoinsGraph subGraph = getSubGraphByAlias(Sets.newHashSet(edge.pkSide().getAlias()));
+ if (!subGraph.match(pattern, Maps.newHashMap())) {
+ unmatched.add(edge);
+ }
+ }
+ return unmatched;
+ }
+
+ /**
+ * Root: TableRef[TEST_KYLIN_FACT]
+ * Edge: TableRef[TEST_KYLIN_FACT] LEFT JOIN TableRef[TEST_ORDER] ON [ORDER_ID] = [ORDER_ID]
+ * Edge: TableRef[TEST_ORDER] LEFT JOIN TableRef[BUYER_ACCOUNT:TEST_ACCOUNT] ON [BUYER_ID] = [ACCOUNT_ID]
+ * Edge: TableRef[TEST_KYLIN_FACT] LEFT JOIN TableRef[TEST_SELLER_TYPE_DIM] ON [SLR_SEGMENT_CD] = [SELLER_TYPE_CD]
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Root: ").append(center);
+ // build next edges
+ getEdgesByFKSide(center).forEach(e -> buildGraphStr(sb, e, 1));
+ return sb.toString();
+ }
+
+ private void buildGraphStr(StringBuilder sb, @NonNull Edge edge, int indent) {
+ sb.append(IntStream.range(0, indent).mapToObj(i -> " ")
+ .collect(Collectors.joining("", "\n", String.valueOf(edge))));
+ // build next edges
+ getEdgesByFKSide(edge.pkSide()).forEach(e -> buildGraphStr(sb, e, indent + 1));
+ }
+
+ private <T> void addIfAbsent(List<T> edges, T edge) {
+ if (edges.contains(edge)) {
+ return;
+ }
+ edges.add(edge);
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
index 492a05723e..773afd993d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/ComputedColumnUtil.java
@@ -51,12 +51,12 @@ import org.apache.kylin.metadata.model.BadModelException;
import org.apache.kylin.metadata.model.BadModelException.CauseType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
-import org.apache.kylin.metadata.model.JoinsGraph;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.alias.AliasDeduce;
import org.apache.kylin.metadata.model.alias.AliasMapping;
import org.apache.kylin.metadata.model.alias.ExpressionComparator;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -329,7 +329,7 @@ public class ComputedColumnUtil {
if (cc.getTableAlias() != null) {
aliasSets.add(cc.getTableAlias());
}
- return model.getJoinsGraph().getSubgraphByAlias(aliasSets);
+ return model.getJoinsGraph().getSubGraphByAlias(aliasSets);
}
public static boolean isSameName(ComputedColumnDesc col1, ComputedColumnDesc col2) {
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
index 4ec5d9d145..65a359e63f 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
@@ -18,13 +18,19 @@
package org.apache.kylin.metadata.model;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Supplier;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.metadata.model.graph.DefaultJoinEdgeMatcher;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.project.NProjectManager;
import org.junit.Assert;
import org.junit.Before;
@@ -54,13 +60,13 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
.innerJoin(new String[] { "TEST_ORDER.ORDER_ID" }, new String[] { "TEST_KYLIN_FACT.ORDER_ID" }).build();
JoinsGraph factIJOrderGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.innerJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" }).build();
- Assert.assertTrue(orderIJFactGraph.match(factIJOrderGraph, new HashMap<String, String>()));
+ Assert.assertTrue(orderIJFactGraph.match(factIJOrderGraph, new HashMap<>()));
JoinsGraph orderLJfactGraph = new MockJoinGraphBuilder(modelDesc, "TEST_ORDER")
.leftJoin(new String[] { "TEST_ORDER.ORDER_ID" }, new String[] { "TEST_KYLIN_FACT.ORDER_ID" }).build();
JoinsGraph factLJorderGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.leftJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" }).build();
- Assert.assertFalse(orderLJfactGraph.match(factLJorderGraph, new HashMap<String, String>()));
+ Assert.assertFalse(orderLJfactGraph.match(factLJorderGraph, new HashMap<>()));
}
@Test
@@ -74,10 +80,10 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
.innerJoin(new String[] { "TEST_ORDER.ORDER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" })
.innerJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_COUNTRY" }, new String[] { "BUYER_COUNTRY.COUNTRY" })
.build();
- Assert.assertFalse(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<String, String>(),
+ Assert.assertFalse(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<>(),
KylinConfig.getInstanceFromEnv().isQueryMatchPartialInnerJoinModel()));
overwriteSystemProp("kylin.query.match-partial-inner-join-model", "true");
- Assert.assertTrue(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<String, String>(),
+ Assert.assertTrue(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<>(),
KylinConfig.getInstanceFromEnv().isQueryMatchPartialInnerJoinModel()));
}
@@ -92,7 +98,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
.innerJoin(new String[] { "TEST_ORDER.ORDER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" })
.innerJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_COUNTRY" }, new String[] { "BUYER_COUNTRY.COUNTRY" })
.build();
- Assert.assertFalse(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<String, String>(),
+ Assert.assertFalse(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<>(),
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default").getConfig()
.isQueryMatchPartialInnerJoinModel()));
@@ -102,7 +108,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
}
});
- Assert.assertTrue(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<String, String>(),
+ Assert.assertTrue(innerJoinGraph.match(innerAndInnerJoinGraph, new HashMap<>(),
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default").getConfig()
.isQueryMatchPartialInnerJoinModel()));
@@ -116,7 +122,8 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
private void overrideProjectConfig(Map<String, String> overrideKylinProps) {
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).updateProject("default", copyForWrite -> {
- copyForWrite.getOverrideKylinProps().putAll(KylinConfig.trimKVFromMap(overrideKylinProps));
+ LinkedHashMap<String, String> map = KylinConfig.trimKVFromMap(overrideKylinProps);
+ copyForWrite.getOverrideKylinProps().putAll(map);
});
}
@@ -135,8 +142,8 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
.innerJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" })
.innerJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_COUNTRY" }, new String[] { "BUYER_COUNTRY.COUNTRY" })
.build();
- Assert.assertTrue(graph1.match(graph2, new HashMap<String, String>()));
- Assert.assertTrue(graph2.match(graph1, new HashMap<String, String>()));
+ Assert.assertTrue(graph1.match(graph2, new HashMap<>()));
+ Assert.assertTrue(graph2.match(graph1, new HashMap<>()));
JoinsGraph graph3 = new MockJoinGraphBuilder(modelDesc, "TEST_ORDER")
.leftJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" })
@@ -147,8 +154,8 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
.leftJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" })
.leftJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_COUNTRY" }, new String[] { "BUYER_COUNTRY.COUNTRY" })
.build();
- Assert.assertTrue(graph3.match(graph4, new HashMap<String, String>()));
- Assert.assertTrue(graph4.match(graph3, new HashMap<String, String>()));
+ Assert.assertTrue(graph3.match(graph4, new HashMap<>()));
+ Assert.assertTrue(graph4.match(graph3, new HashMap<>()));
}
@Test
@@ -158,24 +165,24 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
JoinsGraph modelJoinsGraph = modelDesc.getJoinsGraph();
JoinsGraph singleTblGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT").build();
- Assert.assertTrue(modelJoinsGraph.match(modelJoinsGraph, new HashMap<String, String>()));
- Assert.assertTrue(singleTblGraph.match(singleTblGraph, new HashMap<String, String>()));
- Assert.assertTrue(singleTblGraph.match(modelJoinsGraph, new HashMap<String, String>()));
- Assert.assertFalse(modelJoinsGraph.match(singleTblGraph, new HashMap<String, String>()));
+ Assert.assertTrue(modelJoinsGraph.match(modelJoinsGraph, new HashMap<>()));
+ Assert.assertTrue(singleTblGraph.match(singleTblGraph, new HashMap<>()));
+ Assert.assertTrue(singleTblGraph.match(modelJoinsGraph, new HashMap<>()));
+ Assert.assertFalse(modelJoinsGraph.match(singleTblGraph, new HashMap<>()));
JoinsGraph noFactGraph = new MockJoinGraphBuilder(modelDesc, "TEST_ORDER")
.leftJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }).build();
- Assert.assertFalse(noFactGraph.match(modelJoinsGraph, new HashMap<String, String>()));
+ Assert.assertFalse(noFactGraph.match(modelJoinsGraph, new HashMap<>()));
JoinsGraph factJoinGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.leftJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.leftJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }).build();
- Assert.assertTrue(factJoinGraph.match(modelJoinsGraph, new HashMap<String, String>()));
+ Assert.assertTrue(factJoinGraph.match(modelJoinsGraph, new HashMap<>()));
JoinsGraph joinedFactGraph = new MockJoinGraphBuilder(modelDesc, "BUYER_ACCOUNT")
.leftJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }, new String[] { "TEST_ORDER.BUYER_ID" })
.leftJoin(new String[] { "TEST_ORDER.ORDER_ID" }, new String[] { "TEST_KYLIN_FACT.ORDER_ID" }).build();
- Assert.assertFalse(joinedFactGraph.match(factJoinGraph, new HashMap<String, String>()));
+ Assert.assertFalse(joinedFactGraph.match(factJoinGraph, new HashMap<>()));
}
@Test
@@ -185,24 +192,24 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
JoinsGraph modelJoinsGraph = modelDesc.getJoinsGraph();
JoinsGraph singleTblGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT").build();
- Assert.assertTrue(modelJoinsGraph.match(modelJoinsGraph, new HashMap<String, String>()));
- Assert.assertTrue(singleTblGraph.match(singleTblGraph, new HashMap<String, String>()));
- Assert.assertFalse(singleTblGraph.match(modelJoinsGraph, new HashMap<String, String>()));
- Assert.assertFalse(modelJoinsGraph.match(singleTblGraph, new HashMap<String, String>()));
+ Assert.assertTrue(modelJoinsGraph.match(modelJoinsGraph, new HashMap<>()));
+ Assert.assertTrue(singleTblGraph.match(singleTblGraph, new HashMap<>()));
+ Assert.assertFalse(singleTblGraph.match(modelJoinsGraph, new HashMap<>()));
+ Assert.assertFalse(modelJoinsGraph.match(singleTblGraph, new HashMap<>()));
JoinsGraph noFactGraph = new MockJoinGraphBuilder(modelDesc, "TEST_ORDER")
.innerJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }).build();
- Assert.assertFalse(noFactGraph.match(modelJoinsGraph, new HashMap<String, String>()));
+ Assert.assertFalse(noFactGraph.match(modelJoinsGraph, new HashMap<>()));
JoinsGraph factJoinGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.innerJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.innerJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }).build();
- Assert.assertFalse(factJoinGraph.match(modelJoinsGraph, new HashMap<String, String>()));
+ Assert.assertFalse(factJoinGraph.match(modelJoinsGraph, new HashMap<>()));
JoinsGraph joinedFactGraph = new MockJoinGraphBuilder(modelDesc, "BUYER_ACCOUNT")
.innerJoin(new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }, new String[] { "TEST_ORDER.BUYER_ID" })
.innerJoin(new String[] { "TEST_ORDER.ORDER_ID" }, new String[] { "TEST_KYLIN_FACT.ORDER_ID" }).build();
- Assert.assertTrue(joinedFactGraph.match(factJoinGraph, new HashMap<String, String>()));
+ Assert.assertTrue(joinedFactGraph.match(factJoinGraph, new HashMap<>()));
}
@Test
@@ -214,7 +221,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
JoinsGraph factJoinGraph = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.innerJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.innerJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "BUYER_ACCOUNT.ACCOUNT_ID" }).build();
- Assert.assertTrue(factJoinGraph.match(modelJoinsGraph, new HashMap<String, String>(), true));
+ Assert.assertTrue(factJoinGraph.match(modelJoinsGraph, new HashMap<>(), true));
}
@Test
@@ -229,7 +236,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
JoinsGraph graph2 = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.leftJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.nonEquiLeftJoin("BUYER_ACCOUNT", "TEST_ORDER", "TEST_ORDER.BUYER_ID").build();
- Assert.assertTrue(graph1.match(graph2, new HashMap<String, String>()));
+ Assert.assertTrue(graph1.match(graph2, new HashMap<>()));
}
{
@@ -240,7 +247,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
.leftJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.leftJoin(new String[] { "TEST_ORDER.BUYER_ID" }, new String[] { "SELLER_ACCOUNT.ACCOUNT_ID" })
.nonEquiLeftJoin("BUYER_ACCOUNT", "TEST_ORDER", "TEST_ORDER.BUYER_ID").build();
- Assert.assertTrue(graph1.match(graph2, new HashMap<String, String>()));
+ Assert.assertTrue(graph1.match(graph2, new HashMap<>()));
}
{
@@ -250,7 +257,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
JoinsGraph graph2 = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.leftJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.nonEquiLeftJoin("BUYER_ACCOUNT", "TEST_ORDER", "TEST_ORDER.BUYER_ID").build();
- Assert.assertFalse(graph1.match(graph2, new HashMap<String, String>()));
+ Assert.assertFalse(graph1.match(graph2, new HashMap<>()));
}
{
@@ -258,7 +265,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
JoinsGraph graph2 = new MockJoinGraphBuilder(modelDesc, "TEST_KYLIN_FACT")
.leftJoin(new String[] { "TEST_KYLIN_FACT.ORDER_ID" }, new String[] { "TEST_ORDER.ORDER_ID" })
.nonEquiLeftJoin("BUYER_ACCOUNT", "TEST_ORDER", "TEST_ORDER.BUYER_ID").build();
- Assert.assertFalse(graph1.match(graph2, new HashMap<String, String>()));
+ Assert.assertFalse(graph1.match(graph2, new HashMap<>()));
}
{
@@ -339,4 +346,26 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
Assert.assertTrue(graph1.match(graph2, matchesMapSupplier.get(), true, true));
}
}
+
+ @Test
+ public void testColumnDescEquals() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+ NTableMetadataManager manager = NTableMetadataManager.getInstance(getTestConfig(), "default");
+ TableDesc tableDesc = manager.getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ DefaultJoinEdgeMatcher matcher = new DefaultJoinEdgeMatcher();
+ ColumnDesc one = new ColumnDesc();
+ one.setTable(tableDesc);
+ one.setName("one");
+ ColumnDesc two = new ColumnDesc();
+ two.setTable(tableDesc);
+ two.setName("two");
+ ColumnDesc copyOfOne = new ColumnDesc(one);
+ ColumnDesc copyOfTwo = new ColumnDesc(two);
+ ColumnDesc[] a = new ColumnDesc[] { one, two };
+ ColumnDesc[] b = new ColumnDesc[] { copyOfTwo, copyOfOne };
+ Method method = matcher.getClass().getDeclaredMethod("columnDescEquals", ColumnDesc[].class,
+ ColumnDesc[].class);
+ Unsafe.changeAccessibleObject(method, true);
+ Object invoke = method.invoke(matcher, a, b);
+ Assert.assertEquals(true, invoke);
+ }
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/MockJoinGraphBuilder.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/MockJoinGraphBuilder.java
index 927bc5408c..e47dc65f3a 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/MockJoinGraphBuilder.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/MockJoinGraphBuilder.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.calcite.sql.JoinType;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.junit.Assert;
import com.google.common.collect.Lists;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 9713500115..9473509173 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -43,11 +43,11 @@ import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.realization.HybridRealization;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.JoinsGraph;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryMetrics;
import org.apache.kylin.metadata.realization.IRealization;
@@ -535,7 +535,7 @@ public class OLAPContext {
}
public void matchJoinWithEnhancementTransformation() {
- this.setJoinsGraph(JoinsGraph.normalizeJoinGraph(joinsGraph));
+ joinsGraph.normalize();
}
public RexInputRef createUniqueInputRefContextTables(OLAPTableScan table, int columnIdx) {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index 54f89276bb..21908533a2 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -79,7 +79,6 @@ import org.apache.kylin.metadata.model.FusionModelManager;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.JoinsGraph;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
@@ -88,6 +87,7 @@ import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.project.NProjectLoader;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
index 5e7bc10c46..8cc71a523c 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
@@ -50,12 +50,13 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ColExcludedChecker;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.JoinsGraph;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.alias.ExpressionComparator;
+import org.apache.kylin.metadata.model.graph.DefaultJoinEdgeMatcher;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.relnode.ColumnRowType;
@@ -239,7 +240,7 @@ public class QueryAliasMatcher {
return null;
}
- private static class CCJoinEdgeMatcher extends JoinsGraph.DefaultJoinEdgeMatcher {
+ private static class CCJoinEdgeMatcher extends DefaultJoinEdgeMatcher {
transient QueryAliasMatchInfo matchInfo;
boolean compareCCExpr;
[kylin] 15/38: KYLIN-5530 remove repartition write KYLIN-5530 Optimized snapshot builds KYLIN-5530 Flat Table Repartition before writing data source tables/directories
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 597a9367ec12cfe18c8878a548f07db514ce7e7c
Author: Mingming Ge <7m...@gmail.com>
AuthorDate: Mon Feb 20 14:42:54 2023 +0800
KYLIN-5530 remove repartition write
KYLIN-5530 Optimized snapshot builds
KYLIN-5530 Flat Table Repartition before writing data source tables/directories
---
pom.xml | 2 +-
.../java/org/apache/kylin/common/KapConfig.java | 4 +
.../org/apache/kylin/common/KylinConfigBase.java | 8 ++
.../engine/spark/builder/SegmentFlatTable.scala | 1 +
.../engine/spark/builder/SnapshotBuilder.scala | 11 ++-
.../kylin/engine/spark/job/SegmentExec.scala | 5 +-
.../job/stage/build/FlatTableAndDictBase.scala | 28 +++++--
.../kylin/engine/spark/utils/Repartitioner.java | 15 ++--
.../datasource/storage/LayoutFormatWriter.scala | 98 ++++++++++++++++++++++
.../sql/datasource/storage/StorageStore.scala | 51 +++--------
10 files changed, 160 insertions(+), 63 deletions(-)
diff --git a/pom.xml b/pom.xml
index 944f95c459..53a49fb81c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -124,7 +124,7 @@
<!-- Spark versions -->
<delta.version>1.2.1</delta.version>
- <spark.version>3.2.0-kylin-4.6.5.0</spark.version>
+ <spark.version>3.2.0-kylin-4.6.6.0-SNAPSHOT</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index 4f2132760e..d073f089a7 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -682,6 +682,10 @@ public class KapConfig {
return Boolean.parseBoolean(config.getOptional("kylin.build.optimize-shard-enabled", TRUE));
}
+ public boolean isAggIndexAdaptiveBuildEnabled() {
+ return Boolean.parseBoolean(config.getOptional("kylin.engine.aggIndex-adaptive-build-enabled", FALSE));
+ }
+
public String getSwitchBackupFsExceptionAllowString() {
return config.getOptional("kylin.query.switch-backup-fs-exception-allow-string", "alluxio");
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 0490ce7b84..853c5d6ce6 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2637,10 +2637,18 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.engine.persist-flattable-enabled", TRUE));
}
+ public boolean isFlatTableRedistributionEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.redistribution-flattable-enabled", FALSE));
+ }
+
public boolean isPersistFlatViewEnabled() {
return Boolean.parseBoolean(getOptional("kylin.engine.persist-flatview", FALSE));
}
+ public boolean isPersistFlatUseSnapshotEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.persist-flat-use-snapshot-enabled", TRUE));
+ }
+
public boolean isBuildExcludedTableEnabled() {
return Boolean.parseBoolean(getOptional("kylin.engine.build-excluded-table", FALSE));
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
index 7449aebdcd..e6e41fd9b9 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
@@ -37,6 +37,7 @@ import org.apache.kylin.metadata.model._
import org.apache.kylin.query.util.PushDownUtil
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, expr}
+import org.apache.spark.sql.manager.SparderLookupManager
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.utils.ProxyThreadUtils
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
index 777c16d464..f20c3b855e 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
@@ -431,9 +431,11 @@ class SnapshotBuilder(var jobId: String) extends Logging with Serializable {
if (repartitionNum == 0) {
sourceData.write.parquet(resourcePath)
} else {
- sourceData.repartition(repartitionNum).write.parquet(resourcePath)
+ sourceData.repartition().write.parquet(resourcePath)
}
- val (originSize, totalRows) = computeSnapshotSize(sourceData)
+
+ val snapshotDS = ss.read.parquet(resourcePath)
+ val (originSize, totalRows) = computeSnapshotSize(snapshotDS)
resultMap.put(tableDesc.getIdentity, Result(snapshotTablePath, originSize, totalRows))
}
@@ -458,10 +460,11 @@ class SnapshotBuilder(var jobId: String) extends Logging with Serializable {
List((totalSize, totalRows)).toIterator
}(Encoders.tuple(Encoders.scalaLong, Encoders.scalaLong))
- if (ds.isEmpty) {
+ val stats = ds.collect().reduceOption((a, b) => (a._1 + b._1, a._2 + b._2))
+ if (stats.isEmpty) {
(0L, 0L)
} else {
- ds.reduce((a, b) => (a._1 + b._1, a._2 + b._2))
+ stats.get
}
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala
index 61675ac4a8..0b104eb42a 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala
@@ -270,7 +270,7 @@ trait SegmentExec extends Logging {
require(layout.getIndex.getMeasures.isEmpty)
val dimensions = wrapDimensions(layout)
val columns = NSparkCubingUtil.getColumns(dimensions)
- parentDS.select(columns: _*).sortWithinPartitions(columns: _*)
+ parentDS.select(columns: _*)
}
protected def columnIdFunc(colRef: TblColRef): String
@@ -278,11 +278,10 @@ trait SegmentExec extends Logging {
private def wrapAggLayoutDS(layout: LayoutEntity, parentDS: Dataset[Row]): Dataset[Row] = {
val dimensions = wrapDimensions(layout)
val measures = layout.getOrderedMeasures.keySet()
- val sortColumns = NSparkCubingUtil.getColumns(dimensions)
val selectColumns = NSparkCubingUtil.getColumns(NSparkCubingUtil.combineIndices(dimensions, measures))
val aggregated = CuboidAggregator.aggregate(parentDS, //
dimensions, layout.getIndex.getEffectiveMeasures, columnIdFunc)
- aggregated.select(selectColumns: _*).sortWithinPartitions(sortColumns: _*)
+ aggregated.select(selectColumns: _*)
}
protected final def newDataLayout(segment: NDataSegment, //
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 1409ff8228..a8b28d283b 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -18,9 +18,7 @@
package org.apache.kylin.engine.spark.job.stage.build
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-import java.util.{Locale, Objects, Timer, TimerTask}
-
+import com.google.common.collect.Sets
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.Path
import org.apache.kylin.common.util.HadoopUtil
@@ -54,6 +52,8 @@ import java.util.{Locale, Objects, Timer, TimerTask}
import org.apache.kylin.common.constant.LogConstant
import org.apache.kylin.common.logging.SetLogCategory
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+import java.util.{Locale, Objects, Timer, TimerTask}
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.parallel.ForkJoinTaskSupport
@@ -61,8 +61,6 @@ import scala.concurrent.duration.{Duration, MILLISECONDS}
import scala.concurrent.forkjoin.ForkJoinPool
import scala.util.{Failure, Success, Try}
-import com.google.common.collect.Sets
-
abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
private val dataSegment: NDataSegment,
private val buildParam: BuildParam)
@@ -284,7 +282,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
}
if (joinDesc.isFlattenable && !antiFlattenTableSet.contains(joinDesc.getTable)) {
val tableRef = joinDesc.getTableRef
- val tableDS = newTableDS(tableRef)
+ val tableDS = newSnapshotDS(tableRef)
ret.put(joinDesc, fulfillDS(tableDS, Set.empty, tableRef))
}
}
@@ -353,7 +351,11 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
}
logInfo(s"Segment $segmentId persist flat table: $flatTablePath")
sparkSession.sparkContext.setJobDescription(s"Segment $segmentId persist flat table.")
+ if (config.isFlatTableRedistributionEnabled) {
+ sparkSession.sessionState.conf.setLocalProperty("spark.sql.sources.repartitionWritingDataSource", "true")
+ }
tableDS.write.mode(SaveMode.Overwrite).parquet(flatTablePath.toString)
+ sparkSession.sessionState.conf.setLocalProperty("spark.sql.sources.repartitionWritingDataSource", null)
DFBuilderHelper.checkPointSegment(dataSegment, (copied: NDataSegment) => {
copied.setFlatTableReady(true)
if (dataSegment.isFlatTableReady) {
@@ -420,6 +422,20 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
Some(tableDS)
}
+ def newSnapshotDS(tableRef: TableRef): Dataset[Row] = {
+ val snapshotResPath = tableRef.getTableDesc.getLastSnapshotPath
+ val baseDir = KapConfig.getInstanceFromEnv.getMetadataWorkingDirectory
+ val snapshotResFilePath = new Path(baseDir + snapshotResPath)
+ val fs = HadoopUtil.getWorkingFileSystem
+ if (snapshotResPath == null
+ || !fs.exists(snapshotResFilePath)
+ || config.isPersistFlatUseSnapshotEnabled) {
+ newTableDS(tableRef)
+ } else {
+ sparkSession.read.parquet(snapshotResFilePath.toString).alias(tableRef.getAlias)
+ }
+ }
+
protected def newTableDS(tableRef: TableRef): Dataset[Row] = {
// By design, why not try recovering from table snapshot.
// If fact table is a view and its snapshot exists, that will benefit.
diff --git a/src/spark-project/spark-common/src/main/java/org/apache/kylin/engine/spark/utils/Repartitioner.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/engine/spark/utils/Repartitioner.java
index 95472f5edf..240c55b564 100644
--- a/src/spark-project/spark-common/src/main/java/org/apache/kylin/engine/spark/utils/Repartitioner.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/engine/spark/utils/Repartitioner.java
@@ -18,11 +18,7 @@
package org.apache.kylin.engine.spark.utils;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -35,7 +31,10 @@ import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
public class Repartitioner {
protected static final Logger logger = LoggerFactory.getLogger(Repartitioner.class);
@@ -51,8 +50,8 @@ public class Repartitioner {
private boolean optimizeShardEnabled;
public Repartitioner(int shardSize, int fileLengthThreshold, long totalRowCount, long rowCountThreshold,
- ContentSummary contentSummary, List<Integer> shardByColumns, List<Integer> sortByColumns,
- boolean optimizeShardEnabled) {
+ ContentSummary contentSummary, List<Integer> shardByColumns, List<Integer> sortByColumns,
+ boolean optimizeShardEnabled) {
this.shardSize = shardSize;
this.fileLengthThreshold = fileLengthThreshold;
this.totalRowCount = totalRowCount;
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/LayoutFormatWriter.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/LayoutFormatWriter.scala
new file mode 100644
index 0000000000..3ee230b703
--- /dev/null
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/LayoutFormatWriter.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.datasource.storage
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.kylin.common.KapConfig
+import org.apache.kylin.common.util.HadoopUtil
+import org.apache.kylin.engine.spark.job.NSparkCubingUtil
+import org.apache.kylin.engine.spark.utils.StorageUtils.findCountDistinctMeasure
+import org.apache.kylin.engine.spark.utils.{JobMetrics, Metrics, Repartitioner, StorageUtils}
+import org.apache.kylin.metadata.cube.model.LayoutEntity
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.DataFrame
+
+object LayoutFormatWriter extends Logging {
+
+ protected val TEMP_FLAG = "_temp_"
+
+ /** Describes how output files should be placed in the filesystem. */
+ case class OutputSpec(
+ metrics: JobMetrics,
+ rowCount: Long,
+ hadoopConf: Configuration,
+ bucketNum: Int)
+
+ def write(
+ dataFrame: DataFrame,
+ layout: LayoutEntity,
+ outputPath: Path,
+ kapConfig: KapConfig,
+ storageListener: Option[StorageListener]): OutputSpec = {
+ val ss = dataFrame.sparkSession
+ val hadoopConf = ss.sparkContext.hadoopConfiguration
+ val fs = outputPath.getFileSystem(hadoopConf)
+
+ val dims = layout.getOrderedDimensions.keySet()
+ val sortColumns = NSparkCubingUtil.getColumns(dims)
+
+ if (kapConfig.isAggIndexAdaptiveBuildEnabled
+ && unNeedRepartitionByShardCols(layout)) {
+ val df = dataFrame
+ .repartition()
+ .sortWithinPartitions(sortColumns: _*)
+ val metrics = StorageUtils.writeWithMetrics(df, outputPath.toString)
+ val rowCount = metrics.getMetrics(Metrics.CUBOID_ROWS_CNT)
+ OutputSpec(metrics, rowCount, hadoopConf, -1)
+ } else {
+ val tempPath = outputPath.toString + TEMP_FLAG + System.currentTimeMillis()
+ val df = dataFrame.sortWithinPartitions(sortColumns: _*)
+ val metrics = StorageUtils.writeWithMetrics(df, tempPath)
+ val rowCount = metrics.getMetrics(Metrics.CUBOID_ROWS_CNT)
+ storageListener.foreach(_.onPersistBeforeRepartition(df, layout))
+
+ val bucketNum = StorageUtils.calculateBucketNum(tempPath, layout, rowCount, kapConfig)
+ val summary = HadoopUtil.getContentSummary(fs, new Path(tempPath))
+ val repartitionThresholdSize = if (findCountDistinctMeasure(layout)) {
+ kapConfig.getParquetStorageCountDistinctShardSizeRowCount
+ } else {
+ kapConfig.getParquetStorageShardSizeRowCount
+ }
+
+ val repartitioner = new Repartitioner(
+ kapConfig.getParquetStorageShardSizeMB,
+ kapConfig.getParquetStorageRepartitionThresholdSize,
+ rowCount,
+ repartitionThresholdSize,
+ summary,
+ layout.getShardByColumns,
+ layout.getOrderedDimensions.keySet().asList(),
+ kapConfig.optimizeShardEnabled()
+ )
+ repartitioner.doRepartition(outputPath.toString, tempPath, bucketNum, ss)
+ storageListener.foreach(_.onPersistAfterRepartition(ss.read.parquet(outputPath.toString), layout))
+ OutputSpec(metrics, rowCount, hadoopConf, bucketNum)
+ }
+ }
+
+ def unNeedRepartitionByShardCols(layout: LayoutEntity): Boolean = {
+ layout.getShardByColumns == null || layout.getShardByColumns.isEmpty
+ }
+
+}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/StorageStore.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/StorageStore.scala
index 9f59d2f9dc..4d05368d6e 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/StorageStore.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/datasource/storage/StorageStore.scala
@@ -18,14 +18,14 @@
package org.apache.spark.sql.datasource.storage
-import org.apache.kylin.engine.spark.job.NSparkCubingUtil
-import org.apache.kylin.engine.spark.utils.StorageUtils.findCountDistinctMeasure
-import org.apache.kylin.engine.spark.utils.{JobMetrics, Metrics, Repartitioner, StorageUtils}
-import org.apache.kylin.metadata.cube.model.{LayoutEntity, NDataSegment, NDataflow}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.kylin.common.KapConfig
import org.apache.kylin.common.util.HadoopUtil
+import org.apache.kylin.engine.spark.job.NSparkCubingUtil
+import org.apache.kylin.engine.spark.utils.StorageUtils.findCountDistinctMeasure
+import org.apache.kylin.engine.spark.utils.{Metrics, Repartitioner, StorageUtils}
+import org.apache.kylin.metadata.cube.model.{LayoutEntity, NDataSegment, NDataflow}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.LayoutEntityConverter._
import org.apache.spark.sql.catalyst.catalog.CatalogTable
@@ -92,43 +92,12 @@ abstract class StorageStore extends Logging {
class StorageStoreV1 extends StorageStore {
override def save(layout: LayoutEntity, outputPath: Path, kapConfig: KapConfig, dataFrame: DataFrame): WriteTaskStats = {
- val (metrics: JobMetrics, rowCount: Long, hadoopConf: Configuration, bucketNum: Int) =
- repartitionWriter(layout, outputPath, kapConfig, dataFrame)
- val (fileCount, byteSize) = collectFileCountAndSizeAfterSave(outputPath, hadoopConf)
+ val outputSpec =
+ LayoutFormatWriter.write(dataFrame, layout, outputPath, kapConfig, storageListener)
+ val (fileCount, byteSize) = collectFileCountAndSizeAfterSave(outputPath, outputSpec.hadoopConf)
checkAndWriterFastBitmapLayout(dataFrame, layout, kapConfig, outputPath)
- WriteTaskStats(0, fileCount, byteSize, rowCount, metrics.getMetrics(Metrics.SOURCE_ROWS_CNT), bucketNum, new util.ArrayList[String]())
- }
-
- private def repartitionWriter(layout: LayoutEntity, outputPath: Path, kapConfig: KapConfig, dataFrame: DataFrame) = {
- val ss = dataFrame.sparkSession
- val hadoopConf = ss.sparkContext.hadoopConfiguration
- val fs = outputPath.getFileSystem(hadoopConf)
-
- val tempPath = outputPath.toString + TEMP_FLAG + System.currentTimeMillis()
- val metrics = StorageUtils.writeWithMetrics(dataFrame, tempPath)
- val rowCount = metrics.getMetrics(Metrics.CUBOID_ROWS_CNT)
- storageListener.foreach(_.onPersistBeforeRepartition(dataFrame, layout))
-
- val bucketNum = StorageUtils.calculateBucketNum(tempPath, layout, rowCount, kapConfig)
- val summary = HadoopUtil.getContentSummary(fs, new Path(tempPath))
- val repartitionThresholdSize = if (findCountDistinctMeasure(layout)) {
- kapConfig.getParquetStorageCountDistinctShardSizeRowCount
- } else {
- kapConfig.getParquetStorageShardSizeRowCount
- }
- val repartitioner = new Repartitioner(
- kapConfig.getParquetStorageShardSizeMB,
- kapConfig.getParquetStorageRepartitionThresholdSize,
- rowCount,
- repartitionThresholdSize,
- summary,
- layout.getShardByColumns,
- layout.getOrderedDimensions.keySet().asList(),
- kapConfig.optimizeShardEnabled()
- )
- repartitioner.doRepartition(outputPath.toString, tempPath, bucketNum, ss)
- storageListener.foreach(_.onPersistAfterRepartition(ss.read.parquet(outputPath.toString), layout))
- (metrics, rowCount, hadoopConf, bucketNum)
+ WriteTaskStats(0, fileCount, byteSize, outputSpec.rowCount,
+ outputSpec.metrics.getMetrics(Metrics.SOURCE_ROWS_CNT), outputSpec.bucketNum, new util.ArrayList[String]())
}
def checkAndWriterFastBitmapLayout(dataset: DataFrame, layoutEntity: LayoutEntity, kapConfig: KapConfig, layoutPath: Path): Unit = {
@@ -153,7 +122,7 @@ class StorageStoreV1 extends StorageStore {
}
val afterReplaced = replaceCountDistinctEvalColumn(bitmaps, dataset)
- repartitionWriter(layoutEntity, outputPath, kapConfig, afterReplaced)
+ LayoutFormatWriter.write(afterReplaced, layoutEntity, outputPath, kapConfig, storageListener)
}
[kylin] 14/38: KYLIN-5529 Support adding "basic agg index" or "basic table index" separately
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 620a6ad5bfd6ef06e1bd082deec6bc5e0d74d3cb
Author: Hang Jia <75...@qq.com>
AuthorDate: Fri Feb 24 10:59:06 2023 +0800
KYLIN-5529 Support adding "basic agg index" or "basic table index" separately
---
.../kylin/metadata/cube/model/IndexPlan.java | 28 ++++++++---
.../rest/controller/NModelControllerTest.java | 57 ++++++++++++++++++++++
.../apache/kylin/rest/request/ModelRequest.java | 6 +++
.../kylin/rest/service/BaseIndexUpdateHelper.java | 34 ++++++++++---
.../apache/kylin/rest/service/ModelService.java | 24 +++++++--
.../apache/kylin/rest/service/BaseIndexTest.java | 47 +++++++++++++-----
.../kylin/rest/service/ModelServiceTest.java | 32 +++++++++++-
7 files changed, 197 insertions(+), 31 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
index 33c708b9e8..b2f70ee752 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
@@ -24,6 +24,7 @@ import static org.apache.kylin.metadata.cube.model.IndexEntity.isAggIndex;
import static org.apache.kylin.metadata.cube.model.IndexEntity.isTableIndex;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collection;
@@ -994,16 +995,29 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
return !curLayout.equalsCols(replace);
}
- public void createAndAddBaseIndex(NDataModel model) {
+ public void createAndAddBaseIndex(NDataModel model, List<IndexEntity.Source> sources) {
checkIsNotCachedAndShared();
- LayoutEntity agg = createBaseAggIndex(model);
- LayoutEntity table = createBaseTableIndex(model);
List<LayoutEntity> baseLayouts = Lists.newArrayList();
- baseLayouts.add(agg);
- if (table != null) {
- baseLayouts.add(table);
+ if (sources.contains(IndexEntity.Source.BASE_AGG_INDEX)) {
+ LayoutEntity agg = createBaseAggIndex(model);
+ baseLayouts.add(agg);
+ }
+
+ if (sources.contains(IndexEntity.Source.BASE_TABLE_INDEX)) {
+ LayoutEntity table = createBaseTableIndex(model);
+ if (table != null) {
+ baseLayouts.add(table);
+ }
}
- createAndAddBaseIndex(baseLayouts);
+ if (!baseLayouts.isEmpty()) {
+ createAndAddBaseIndex(baseLayouts);
+ }
+ }
+
+ public void createAndAddBaseIndex(NDataModel model) {
+ ArrayList<IndexEntity.Source> sources = Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX,
+ IndexEntity.Source.BASE_TABLE_INDEX);
+ createAndAddBaseIndex(model, sources);
}
public void createAndAddBaseIndex(List<LayoutEntity> needCreateBaseLayouts) {
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
index 47520fa814..1638aa0bad 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
@@ -360,6 +360,63 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class));
}
+ @Test
+ public void testCreateModelWithBaseIndexTypeAndWithBaseIndex() throws Exception {
+ ModelRequest request = new ModelRequest();
+ request.setProject("default");
+
+ request.setBaseIndexType(
+ Sets.newHashSet(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX));
+ request.setWithBaseIndex(true);
+ NDataModel mockModel = new NDataModel();
+ mockModel.setUuid("mock");
+ mockModel.setProject("default");
+ Mockito.doReturn(mockModel).when(modelService).createModel(request.getProject(), request);
+ Mockito.doReturn(new IndexPlan()).when(modelService).getIndexPlan(mockModel.getId(), mockModel.getProject());
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/models").contentType(MediaType.APPLICATION_JSON)
+ .content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class));
+ }
+
+ @Test
+ public void testCreateModelWithBaseIndex() throws Exception {
+ ModelRequest request = new ModelRequest();
+ request.setProject("default");
+
+ request.setWithBaseIndex(true);
+ NDataModel mockModel = new NDataModel();
+ mockModel.setUuid("mock");
+ mockModel.setProject("default");
+ Mockito.doReturn(mockModel).when(modelService).createModel(request.getProject(), request);
+ Mockito.doReturn(new IndexPlan()).when(modelService).getIndexPlan(mockModel.getId(), mockModel.getProject());
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/models").contentType(MediaType.APPLICATION_JSON)
+ .content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class));
+ }
+
+ @Test
+ public void testCreateModelWithBaseIndexType() throws Exception {
+ ModelRequest request = new ModelRequest();
+ request.setProject("default");
+
+ request.setBaseIndexType(
+ Sets.newHashSet(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX));
+ NDataModel mockModel = new NDataModel();
+ mockModel.setUuid("mock");
+ mockModel.setProject("default");
+ Mockito.doReturn(mockModel).when(modelService).createModel(request.getProject(), request);
+ Mockito.doReturn(new IndexPlan()).when(modelService).getIndexPlan(mockModel.getId(), mockModel.getProject());
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/models").contentType(MediaType.APPLICATION_JSON)
+ .content(JsonUtil.writeValueAsString(request))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(nModelController).createModel(Mockito.any(ModelRequest.class));
+ }
+
@Test
public void checkPartitionDesc() throws Exception {
PartitionDesc partitionDesc = new PartitionDesc();
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index 054b7e21f0..e585a8cf47 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -18,10 +18,13 @@
package org.apache.kylin.rest.request;
+
import java.util.Collection;
import java.util.List;
+import java.util.Set;
import java.util.function.BiFunction;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.insensitive.ModelInsensitiveRequest;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -84,6 +87,9 @@ public class ModelRequest extends NDataModel implements ModelInsensitiveRequest
@JsonProperty("with_base_index")
private boolean withBaseIndex = false;
+ @JsonProperty("base_index_type")
+ private Set<IndexEntity.Source> baseIndexType;
+
@JsonProperty("with_second_storage")
private boolean withSecondStorage = false;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java
index 94e0915750..881f77a14f 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/BaseIndexUpdateHelper.java
@@ -17,14 +17,19 @@
*/
package org.apache.kylin.rest.service;
+import java.util.List;
+
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.rest.util.SpringContext;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.rest.request.CreateBaseIndexRequest;
import org.apache.kylin.rest.response.BuildBaseIndexResponse;
+import org.apache.kylin.rest.util.SpringContext;
+
+import com.google.common.collect.Lists;
import io.kyligence.kap.secondstorage.SecondStorageUpdater;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
@@ -43,14 +48,25 @@ public class BaseIndexUpdateHelper {
private String project;
private String modelId;
- private boolean createIfNotExist;
+ private List<IndexEntity.Source> updateBaseIndexTypes;
private boolean needUpdate;
private boolean secondStorageEnabled = false;
@Setter
private boolean needCleanSecondStorage = true;
public BaseIndexUpdateHelper(NDataModel model, boolean createIfNotExist) {
+ this(model, updateTypesByFlag(createIfNotExist));
+ }
+
+ private static List<IndexEntity.Source> updateTypesByFlag(boolean createIfNotExist) {
+ if (createIfNotExist) {
+ return Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX);
+ } else {
+ return Lists.newArrayList();
+ }
+ }
+ public BaseIndexUpdateHelper(NDataModel model, List<IndexEntity.Source> updateBaseIndexTypes) {
NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(),
model.getProject());
IndexPlan indexPlan = indexPlanManager.getIndexPlan(model.getId());
@@ -61,7 +77,7 @@ public class BaseIndexUpdateHelper {
if (needUpdate) {
project = model.getProject();
modelId = model.getId();
- this.createIfNotExist = createIfNotExist;
+ this.updateBaseIndexTypes = updateBaseIndexTypes;
preBaseAggLayout = getBaseAggLayout();
preBaseTableLayout = getBaseTableLayout();
}
@@ -76,18 +92,20 @@ public class BaseIndexUpdateHelper {
if (!needUpdate) {
return BuildBaseIndexResponse.EMPTY;
}
- if (notExist(preBaseAggLayout) && notExist(preBaseTableLayout) && !createIfNotExist) {
+ if (notExist(preBaseAggLayout) && notExist(preBaseTableLayout)
+ && !updateBaseIndexTypes.contains(IndexEntity.Source.BASE_TABLE_INDEX)
+ && !updateBaseIndexTypes.contains(IndexEntity.Source.BASE_AGG_INDEX)) {
return BuildBaseIndexResponse.EMPTY;
}
long curBaseTableLayout = getBaseTableLayout();
- boolean needCreateBaseTable = createIfNotExist;
+ boolean needCreateBaseTable = updateBaseIndexTypes.contains(IndexEntity.Source.BASE_TABLE_INDEX);
if (exist(preBaseTableLayout) && notExist(curBaseTableLayout)) {
needCreateBaseTable = true;
}
Long curExistBaseAggLayout = getBaseAggLayout();
- boolean needCreateBaseAgg = createIfNotExist;
+ boolean needCreateBaseAgg = updateBaseIndexTypes.contains(IndexEntity.Source.BASE_AGG_INDEX);
if (exist(preBaseAggLayout) && notExist(curExistBaseAggLayout)) {
needCreateBaseAgg = true;
}
@@ -98,8 +116,8 @@ public class BaseIndexUpdateHelper {
CreateBaseIndexRequest indexRequest = new CreateBaseIndexRequest();
indexRequest.setModelId(modelId);
indexRequest.setProject(project);
- BuildBaseIndexResponse response = service.updateBaseIndex(project, indexRequest, needCreateBaseAgg,
- needCreateBaseTable, true);
+ BuildBaseIndexResponse response = service.updateBaseIndex(project, indexRequest, needCreateBaseTable,
+ needCreateBaseAgg, true);
response.judgeIndexOperateType(exist(preBaseAggLayout), true);
response.judgeIndexOperateType(exist(preBaseTableLayout), false);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 9a917fb61c..6c5aced159 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -2053,14 +2053,30 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public void addBaseIndex(ModelRequest modelRequest, NDataModel model, IndexPlan indexPlan) {
- if (!modelRequest.isWithSecondStorage() && NDataModel.ModelType.BATCH == model.getModelType()
- && modelRequest.isWithBaseIndex()) {
- indexPlan.createAndAddBaseIndex(model);
+ if (!modelRequest.isWithSecondStorage() && NDataModel.ModelType.BATCH == model.getModelType()) {
+ List<IndexEntity.Source> sources = needHandleBaseIndexType(modelRequest);
+ indexPlan.createAndAddBaseIndex(model, sources);
} else if (modelRequest.isWithSecondStorage()) {
indexPlan.createAndAddBaseIndex(Collections.singletonList(indexPlan.createBaseTableIndex(model)));
}
}
+ private List<IndexEntity.Source> needHandleBaseIndexType(ModelRequest modelRequest) {
+ List<IndexEntity.Source> sources = Lists.newArrayList();
+ if (modelRequest.getBaseIndexType() != null) {
+ if (modelRequest.getBaseIndexType().contains(IndexEntity.Source.BASE_AGG_INDEX)) {
+ sources.add(IndexEntity.Source.BASE_AGG_INDEX);
+ }
+ if (modelRequest.getBaseIndexType().contains(IndexEntity.Source.BASE_TABLE_INDEX)) {
+ sources.add(IndexEntity.Source.BASE_TABLE_INDEX);
+ }
+ } else if (modelRequest.isWithBaseIndex()) {
+ sources.add(IndexEntity.Source.BASE_AGG_INDEX);
+ sources.add(IndexEntity.Source.BASE_TABLE_INDEX);
+ }
+ return sources;
+ }
+
// for streaming & fusion model
private void createStreamingJob(String project, NDataModel model, ModelRequest request) {
if (NDataModel.ModelType.BATCH != model.getModelType()) {
@@ -2995,7 +3011,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
copyModel.init(modelManager.getConfig(), project, modelManager.getCCRelatedModels(copyModel));
BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(originModel,
- request.isWithBaseIndex());
+ needHandleBaseIndexType(request));
preProcessBeforeModelSave(copyModel, project);
val updated = modelManager.updateDataModelDesc(copyModel);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
index 928c903239..8abe394632 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
@@ -142,6 +142,25 @@ public class BaseIndexTest extends SourceTestCase {
compareBaseIndex(getModelIdFrom(modelRequest.getAlias()), baseTableLayout, baseAggLayout);
}
+ @Test
+ public void testCreateBaseTableLayoutWithBaseIndexTypes() {
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
+ ModelRequest modelRequest = FormModel(modelManager.getDataModelDesc(COMMON_MODEL_ID));
+ modelRequest.setWithBaseIndex(false);
+ String modelId = modelService.createModel(modelRequest.getProject(), modelRequest).getId();
+ BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(modelRequest,
+ Lists.newArrayList(Source.BASE_AGG_INDEX));
+ BuildBaseIndexResponse baseIndexResponse = baseIndexUpdater.update(indexPlanService);
+ LayoutEntity baseAggLayout = LayoutBuilder.builder().colOrder(0, 1, 2, 3, 100000, 100001).id(1L).build();
+ LayoutEntity baseTableLayout = null;
+ compareBaseIndex(getModelIdFrom(modelRequest.getAlias()), baseTableLayout, baseAggLayout);
+
+ baseIndexUpdater = new BaseIndexUpdateHelper(modelRequest, Lists.newArrayList(Source.BASE_TABLE_INDEX));
+ baseIndexResponse = baseIndexUpdater.update(indexPlanService);
+ baseTableLayout = LayoutBuilder.builder().colOrder(0, 1, 2, 3).id(20000000001L).build();
+ compareBaseIndex(getModelIdFrom(modelRequest.getAlias()), baseTableLayout, baseAggLayout);
+ }
+
@Test
public void testCreateBaseLayoutWithProperties() {
// create base index is same with index in rulebaseindex or indexes
@@ -402,7 +421,7 @@ public class BaseIndexTest extends SourceTestCase {
IndexPlan indexPlan = NIndexPlanManager.getInstance(getTestConfig(), getProject()).getIndexPlan(modelId);
return indexPlan.getAllLayouts().stream()
.filter(layoutEntity -> layoutEntity.isBase() && isAggIndex(layoutEntity.getId())).findFirst()
- .orElseGet(null);
+ .orElse(null);
}
private String createBaseIndexFromModel(String modelId) {
@@ -417,18 +436,24 @@ public class BaseIndexTest extends SourceTestCase {
if (expectedBaseTableLayout == null && baseTableLayout == null) {
return;
}
- Assert.assertThat(baseAggLayout.getColOrder(), equalTo(expectedBaseAggLayout.getColOrder()));
- Assert.assertThat(baseAggLayout.getShardByColumns(), equalTo(expectedBaseAggLayout.getShardByColumns()));
- Assert.assertThat(baseAggLayout.getSortByColumns(), equalTo(expectedBaseAggLayout.getSortByColumns()));
- Assert.assertThat(baseTableLayout.getColOrder(), equalTo(expectedBaseTableLayout.getColOrder()));
- Assert.assertThat(baseTableLayout.getShardByColumns(), equalTo(expectedBaseTableLayout.getShardByColumns()));
- Assert.assertThat(baseTableLayout.getSortByColumns(), equalTo(expectedBaseTableLayout.getSortByColumns()));
- if (baseAggLayout.getId() != -1) {
- Assert.assertEquals(expectedBaseAggLayout.getId(), baseAggLayout.getId());
+ if (expectedBaseAggLayout != null) {
+ Assert.assertThat(baseAggLayout.getColOrder(), equalTo(expectedBaseAggLayout.getColOrder()));
+ Assert.assertThat(baseAggLayout.getShardByColumns(), equalTo(expectedBaseAggLayout.getShardByColumns()));
+ Assert.assertThat(baseAggLayout.getSortByColumns(), equalTo(expectedBaseAggLayout.getSortByColumns()));
+ if (baseAggLayout.getId() != -1) {
+ Assert.assertEquals(expectedBaseAggLayout.getId(), baseAggLayout.getId());
+ }
}
- if (baseTableLayout.getId() != -1) {
- Assert.assertEquals(expectedBaseTableLayout.getId(), baseTableLayout.getId());
+ if (expectedBaseTableLayout != null) {
+ Assert.assertThat(baseTableLayout.getColOrder(), equalTo(expectedBaseTableLayout.getColOrder()));
+ Assert.assertThat(baseTableLayout.getShardByColumns(),
+ equalTo(expectedBaseTableLayout.getShardByColumns()));
+ Assert.assertThat(baseTableLayout.getSortByColumns(), equalTo(expectedBaseTableLayout.getSortByColumns()));
+
+ if (baseTableLayout.getId() != -1) {
+ Assert.assertEquals(expectedBaseTableLayout.getId(), baseTableLayout.getId());
+ }
}
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index b69b89585b..58971f625c 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -5009,8 +5009,38 @@ public class ModelServiceTest extends SourceTestCase {
when(modelRequest.isWithSecondStorage()).thenReturn(false);
when(model.getModelType()).thenReturn(NDataModel.ModelType.BATCH);
when(modelRequest.isWithBaseIndex()).thenReturn(true);
+ when(modelRequest.getBaseIndexType()).thenReturn(null);
modelService.addBaseIndex(modelRequest, model, indexPlan);
- Mockito.verify(indexPlan).createAndAddBaseIndex(model);
+ Mockito.verify(indexPlan).createAndAddBaseIndex(model,
+ Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX));
+ when(modelRequest.isWithSecondStorage()).thenReturn(true);
+ when(indexPlan.createBaseTableIndex(model)).thenReturn(null);
+ modelService.addBaseIndex(modelRequest, model, indexPlan);
+ Mockito.verify(indexPlan).createAndAddBaseIndex(anyList());
+ }
+
+ @Test
+ public void testAddBaseAggIndex() {
+ val modelRequest = mock(ModelRequest.class);
+ val model = mock(NDataModel.class);
+ val indexPlan = mock(IndexPlan.class);
+
+ when(modelRequest.isWithSecondStorage()).thenReturn(false);
+ when(model.getModelType()).thenReturn(NDataModel.ModelType.BATCH);
+ when(modelRequest.isWithBaseIndex()).thenReturn(true);
+ when(modelRequest.getBaseIndexType()).thenReturn(null);
+ modelService.addBaseIndex(modelRequest, model, indexPlan);
+ Mockito.verify(indexPlan).createAndAddBaseIndex(model,
+ Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX, IndexEntity.Source.BASE_TABLE_INDEX));
+
+ when(modelRequest.getBaseIndexType()).thenReturn(Collections.singleton(IndexEntity.Source.BASE_AGG_INDEX));
+ modelService.addBaseIndex(modelRequest, model, indexPlan);
+ Mockito.verify(indexPlan).createAndAddBaseIndex(model, Lists.newArrayList(IndexEntity.Source.BASE_AGG_INDEX));
+
+ when(modelRequest.getBaseIndexType()).thenReturn(Collections.singleton(IndexEntity.Source.BASE_TABLE_INDEX));
+ modelService.addBaseIndex(modelRequest, model, indexPlan);
+ Mockito.verify(indexPlan).createAndAddBaseIndex(model, Lists.newArrayList(IndexEntity.Source.BASE_TABLE_INDEX));
+
when(modelRequest.isWithSecondStorage()).thenReturn(true);
when(indexPlan.createBaseTableIndex(model)).thenReturn(null);
modelService.addBaseIndex(modelRequest, model, indexPlan);
[kylin] 24/38: KYLIN-5535 Fix the username suffix matching
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9bf63e01838fee5eb2fc50332c43a14bc0ea824d
Author: Jiale He <35...@users.noreply.github.com>
AuthorDate: Tue Feb 28 10:11:04 2023 +0800
KYLIN-5535 Fix the username suffix matching
---
.../kylin/metadata/user/NKylinUserManager.java | 20 +++-----------------
.../kylin/metadata/user/NKylinUserManagerTest.java | 12 ++++++++++++
2 files changed, 15 insertions(+), 17 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
index 4b487a6841..ea0b184068 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
@@ -23,7 +23,6 @@ import static org.apache.kylin.common.persistence.ResourceStore.USER_ROOT;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.NavigableSet;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -92,7 +91,8 @@ public class NKylinUserManager {
return user;
}
return Objects.nonNull(user) ? user
- : crud.listPartial(path -> StringUtils.endsWithIgnoreCase(path, name)).stream().findAny().orElse(null);
+ : crud.listPartial(path -> StringUtils.endsWithIgnoreCase(path, name)).stream()
+ .filter(u -> StringUtils.equalsIgnoreCase(u.getUsername(), name)).findAny().orElse(null);
}
public List<ManagedUser> list() {
@@ -122,21 +122,7 @@ public class NKylinUserManager {
}
public boolean exists(String username) {
- if (StringUtils.isEmpty(username)) {
- return false;
- }
- ManagedUser user = crud.get(username);
- if (getConfig().isMetadataKeyCaseInSensitiveEnabled()) {
- return Objects.nonNull(user);
- }
- if (Objects.nonNull(user)) {
- return true;
- }
- NavigableSet<String> users = getStore().listResources(USER_ROOT);
- if (Objects.isNull(users)) {
- return false;
- }
- return users.stream().anyMatch(path -> StringUtils.endsWithIgnoreCase(path, username));
+ return Objects.nonNull(get(username));
}
public Set<String> getUserGroups(String userName) {
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/user/NKylinUserManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/user/NKylinUserManagerTest.java
index c5246228dd..6f57b71d17 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/user/NKylinUserManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/user/NKylinUserManagerTest.java
@@ -68,4 +68,16 @@ class NKylinUserManagerTest extends NLocalFileMetadataTestCase {
Assertions.assertNull(manager.get("notexist"));
Assertions.assertNull(manager.get(null));
}
+
+ @Test
+ void testNameSuffix() {
+ NKylinUserManager manager = NKylinUserManager.getInstance(getTestConfig());
+ ManagedUser normalUser = new ManagedUser("test_ut", "KYLIN", false, Arrays.asList(
+ new SimpleGrantedAuthority(Constant.ROLE_ANALYST), new SimpleGrantedAuthority(Constant.ROLE_MODELER)));
+ manager.update(normalUser);
+ Assertions.assertTrue(manager.exists("test_ut"));
+ Assertions.assertFalse(manager.exists("ut"));
+ Assertions.assertNotNull(manager.get("test_ut"));
+ Assertions.assertNull(manager.get("ut"));
+ }
}
[kylin] 31/38: KYLIN-5523 [FOLLOW UP] fix generate flat table sql
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6309697701c6ec984a8120d020e9d6c2f947da46
Author: Jiale He <96...@qq.com>
AuthorDate: Fri Mar 3 19:07:30 2023 +0800
KYLIN-5523 [FOLLOW UP] fix generate flat table sql
---
.../src/main/java/org/apache/kylin/query/util/PushDownUtil.java | 3 ++-
.../test/java/org/apache/kylin/query/util/PushDownUtilTest.java | 8 ++++----
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 2f0558d246..4f7f467e17 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -244,7 +244,8 @@ public class PushDownUtil {
sqlBuilder.append(allColStr);
}
- sqlBuilder.append("FROM ").append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity());
+ sqlBuilder.append("FROM ").append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity())
+ .append(" as ").append(StringHelper.doubleQuote(model.getRootFactTable().getAlias()));
appendJoinStatement(model, sqlBuilder, singleLine);
sqlBuilder.append("WHERE ").append(sep);
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index 804fd81ab7..379af78e8e 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -162,7 +162,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ "WHERE\n" //
@@ -198,7 +198,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ "ON SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n"
+ "WHERE\n" //
@@ -232,7 +232,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ "WHERE\n" //
@@ -265,7 +265,7 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+ ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ "WHERE\n" //
[kylin] 36/38: mirror: fix UT for snapshot building
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e19d2918c56506abeeddbb5e5cdc8db4e9cc4e1a
Author: ChenliangLu <31...@users.noreply.github.com>
AuthorDate: Thu Mar 9 16:46:00 2023 +0800
mirror: fix UT for snapshot building
---
.../src/test/scala/org/apache/kylin/it/TestQueryAndBuildFunSuite.scala | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/it/TestQueryAndBuildFunSuite.scala b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/it/TestQueryAndBuildFunSuite.scala
index f4fbd16b7d..469ee85fe5 100644
--- a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/it/TestQueryAndBuildFunSuite.scala
+++ b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/it/TestQueryAndBuildFunSuite.scala
@@ -139,6 +139,7 @@ class TestQueryAndBuildFunSuite
// test for snapshot cleanup
KylinConfig.getInstanceFromEnv.setProperty("kylin.snapshot.version-ttl", "0")
KylinConfig.getInstanceFromEnv.setProperty("kylin.snapshot.max-versions", "1")
+ KylinConfig.getInstanceFromEnv.setProperty("kylin.engine.persist-flat-use-snapshot-enabled", "false")
build()
}
[kylin] 11/38: KYLIN-5527 Data count check for segment layout building
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 59a2e21034f7ccbf859e3aee94668a5ce8a4ec1a
Author: Yinghao Lin <39...@users.noreply.github.com>
AuthorDate: Tue Feb 21 11:35:14 2023 +0800
KYLIN-5527 Data count check for segment layout building
---
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 12 +
.../org/apache/kylin/common/util/JsonUtil.java | 1 +
.../apache/kylin/job/constant/JobStatusEnum.java | 6 +
.../kylin/job/execution/AbstractExecutable.java | 10 +-
.../kylin/job/execution/DefaultExecutable.java | 45 ++-
.../kylin/job/execution/ExecutableState.java | 11 +-
.../apache/kylin/job/execution/ExecuteResult.java | 23 +-
.../kylin/job/execution/NExecutableManager.java | 7 +
.../job/execution/NExecutableManagerTest.java | 21 +
.../kylin/metadata/cube/model/NBatchConstants.java | 1 +
.../kylin/metadata/cube/model/NDataLayout.java | 19 +-
.../kylin/metadata/cube/model/NDataSegDetails.java | 33 +-
.../kylin/metadata/cube/model/NDataSegment.java | 63 ++-
.../metadata/cube/model/NDataflowManager.java | 4 +-
.../metadata/cube/model/SegmentPartition.java | 2 +-
.../org/apache/kylin/metadata/model/ISegment.java | 2 +
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metrics/HdfsCapacityMetricsTest.java | 4 +-
.../kylin/rest/response/ExecutableResponse.java | 6 +-
.../rest/response/ExecutableStepResponse.java | 3 +
.../org/apache/kylin/rest/service/JobService.java | 8 +-
.../org/apache/kylin/rest/service/StageTest.java | 3 +
.../metadata/_global/project/index_build_test.json | 40 ++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 70 ++++
.../b2f206e1-7a15-c94a-20f5-f608d550ead6.json | 69 ++++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 204 ++++++++++
.../3ec47efc-573a-9304-4405-8e05ae184322.json | 241 +++++++++++
.../index_build_test/table/SSB.CUSTOMER.json | 77 ++++
.../index_build_test/table/SSB.LINEORDER.json | 131 ++++++
.../rest/controller/NIndexPlanController.java | 7 +-
.../apache/kylin/rest/response/IndexResponse.java | 4 +
.../kylin/rest/response/NDataSegmentResponse.java | 4 +-
.../kylin/rest/service/FusionIndexService.java | 24 +-
.../kylin/rest/service/IndexPlanService.java | 50 ++-
.../apache/kylin/rest/service/ModelService.java | 4 +-
.../rest/service/params/IndexPlanParams.java} | 31 +-
.../rest/service/params/PaginationParams.java} | 27 +-
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/query/engine/AsyncQueryJobTest.java | 2 +-
src/spark-project/engine-spark/pom.xml | 5 +
.../engine/spark/application/SparkApplication.java | 17 +-
.../builder/PartitionDictionaryBuilderHelper.java | 2 +-
.../spark/job/ExecutableAddCuboidHandler.java | 46 ++-
.../kylin/engine/spark/job/NSparkCubingStep.java | 42 +-
.../spark/merger/AfterBuildResourceMerger.java | 4 +-
.../merger/AfterMergeOrRefreshResourceMerger.java | 4 +-
.../spark/builder/DictionaryBuilderHelper.java | 2 +-
.../kylin/engine/spark/job/BuildJobInfos.scala | 21 +-
.../apache/kylin/engine/spark/job/DFMergeJob.java | 2 +-
.../kylin/engine/spark/job/PartitionExec.scala | 4 +
.../kylin/engine/spark/job/SegmentBuildJob.java | 14 +-
.../kylin/engine/spark/job/SegmentExec.scala | 38 +-
.../apache/kylin/engine/spark/job/SegmentJob.java | 4 +
.../kylin/engine/spark/job/SegmentMergeJob.java | 3 +-
.../kylin/engine/spark/job/stage/StageExec.scala | 113 +++---
.../engine/spark/job/stage/build/BuildLayer.scala | 23 +-
.../engine/spark/job/stage/build/BuildStage.scala | 6 -
.../job/stage/build/GatherFlatTableStats.scala | 2 -
.../spark/job/stage/build/GenerateFlatTable.scala | 145 ++++++-
.../build/partition/PartitionBuildLayer.scala | 23 +-
.../build/partition/PartitionBuildStage.scala | 6 -
.../engine/spark/job/stage/merge/MergeStage.scala | 2 +-
.../merge/partition/PartitionMergeStage.scala | 2 +-
.../GenerateFlatTableWithSparkSessionTest.java | 444 +++++++++++++++++++++
.../kylin/tool/garbage/DataflowCleanerCLI.java | 2 +-
68 files changed, 1976 insertions(+), 285 deletions(-)
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
index b0f9142756..7fc2665b6f 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
@@ -596,14 +596,14 @@ public class AccessServiceTest extends NLocalFileMetadataTestCase {
@Test
public void testGetGrantedProjectsOfUser() throws IOException {
List<String> result = accessService.getGrantedProjectsOfUser("ADMIN");
- assertEquals(28, result.size());
+ assertEquals(29, result.size());
}
@Test
public void testGetGrantedProjectsOfUserOrGroup() throws IOException {
// admin user
List<String> result = accessService.getGrantedProjectsOfUserOrGroup("ADMIN", true);
- assertEquals(28, result.size());
+ assertEquals(29, result.size());
// normal user
result = accessService.getGrantedProjectsOfUserOrGroup("ANALYST", true);
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
index 9a0ee62d7d..20e40e74e6 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
@@ -1316,7 +1316,7 @@ public class AclTCRServiceTest extends NLocalFileMetadataTestCase {
Mockito.when(userService.isGlobalAdmin("ADMIN")).thenReturn(true);
List<SidPermissionWithAclResponse> responses = accessService.getUserOrGroupAclPermissions(projects, "ADMIN",
true);
- Assert.assertEquals(28, responses.size());
+ Assert.assertEquals(29, responses.size());
Assert.assertTrue(responses.stream().allMatch(response -> "ADMIN".equals(response.getProjectPermission())));
// test normal group
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f20dca320b..32af04760a 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -980,6 +980,10 @@ public abstract class KylinConfigBase implements Serializable {
return !getBuildConf().isEmpty() && !getWritingClusterWorkingDir().isEmpty();
}
+ public String getWriteClusterWorkingDir() {
+ return getOptional("kylin.env.write-hdfs-working-dir", "");
+ }
+
public String getWritingClusterWorkingDir() {
return getOptional(WRITING_CLUSTER_WORKING_DIR, "");
}
@@ -3800,6 +3804,14 @@ public abstract class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.source.ddl.logical-view-catchup-interval", "60"));
}
+ public boolean isDataCountCheckEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.build.data-count-check-enabled", FALSE));
+ }
+
+ public boolean isNonStrictCountCheckAllowed() {
+ return Boolean.parseBoolean(getOptional("kylin.build.allow-non-strict-count-check", FALSE));
+ }
+
// ============================================================================
// Cost based index Planner
// ============================================================================
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index d74c7ca1d1..fb0b46826c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -60,6 +60,7 @@ public class JsonUtil {
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+ .configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.setConfig(mapper.getSerializationConfig().withView(PersistenceView.class));
mapper.setFilterProvider(simpleFilterProvider);
indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true)
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java b/src/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
index 73f634bc85..59b0d14934 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java
@@ -100,6 +100,12 @@ public enum JobStatusEnum {
public boolean checkAction(JobActionEnum actionEnum) {
return false;
}
+ },
+ WARNING(2048) {
+ @Override
+ public boolean checkAction(JobActionEnum actionEnum) {
+ return false;
+ }
};
public abstract boolean checkAction(JobActionEnum actionEnum);
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index bfe6f1fb6a..0f8de4e055 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -285,7 +285,9 @@ public abstract class AbstractExecutable implements Executable {
MetricsGroup.hostTagCounterInc(MetricsName.JOB_STEP_ATTEMPTED, MetricsCategory.PROJECT, project, retry);
if (result.succeed()) {
wrapWithCheckQuit(() -> {
- updateJobOutput(project, getId(), ExecutableState.SUCCEED, result.getExtraInfo(), result.output(),
+ ExecutableState state = adjustState(ExecutableState.SUCCEED);
+ logger.info("Job {} adjust future state from {} to {}", getId(), ExecutableState.SUCCEED.name(), state.name());
+ updateJobOutput(project, getId(), state, result.getExtraInfo(), result.output(),
null);
});
} else if (result.skip()) {
@@ -304,6 +306,10 @@ public abstract class AbstractExecutable implements Executable {
}
}
+ protected ExecutableState adjustState(ExecutableState originalState) {
+ return originalState;
+ }
+
protected void onExecuteStopHook() {
onExecuteErrorHook(getId());
}
@@ -337,7 +343,7 @@ public abstract class AbstractExecutable implements Executable {
//The output will be stored in HDFS,not in RS
if (this instanceof ChainedStageExecutable) {
- if (newStatus == ExecutableState.SUCCEED) {
+ if (newStatus.isNotBad()) {
executableManager.makeStageSuccess(jobId);
} else if (newStatus == ExecutableState.ERROR) {
executableManager.makeStageError(jobId);
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutable.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutable.java
index 5a4e92e58c..78cb903ad1 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutable.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutable.java
@@ -184,7 +184,7 @@ public class DefaultExecutable extends AbstractExecutable implements ChainedExec
private void executeStep(Executable executable, ExecutableContext context) throws ExecuteException {
if (executable.isRunnable()) {
executable.execute(context);
- } else if (ExecutableState.SUCCEED == executable.getStatus()) {
+ } else if (executable.getStatus().isNotBad()) {
logger.info("step {} is already succeed, skip it.", executable.getDisplayName());
} else {
throw new IllegalStateException("invalid subtask state, sub task:" + executable.getDisplayName()
@@ -289,27 +289,28 @@ public class DefaultExecutable extends AbstractExecutable implements ChainedExec
logger.info("Sub-task finished {}, state: {}", task.getDisplayName(), task.getStatus());
boolean taskSucceed = false;
switch (task.getStatus()) {
- case RUNNING:
- hasError = true;
- break;
- case ERROR:
- hasError = true;
- break;
- case DISCARDED:
- hasDiscarded = true;
- break;
- case SUICIDAL:
- hasSuicidal = true;
- break;
- case PAUSED:
- hasPaused = true;
- break;
- case SKIP:
- case SUCCEED:
- taskSucceed = true;
- break;
- default:
- break;
+ case RUNNING:
+ hasError = true;
+ break;
+ case ERROR:
+ hasError = true;
+ break;
+ case DISCARDED:
+ hasDiscarded = true;
+ break;
+ case SUICIDAL:
+ hasSuicidal = true;
+ break;
+ case PAUSED:
+ hasPaused = true;
+ break;
+ case SUCCEED:
+ case SKIP:
+ case WARNING:
+ taskSucceed = true;
+ break;
+ default:
+ break;
}
allSucceed &= taskSucceed;
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
index 9016ee8a40..40999107ce 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java
@@ -33,7 +33,7 @@ import com.google.common.collect.Multimaps;
*/
public enum ExecutableState {
- READY, RUNNING, ERROR, PAUSED, DISCARDED, SUCCEED, SUICIDAL, SKIP;
+ READY, RUNNING, ERROR, PAUSED, DISCARDED, SUCCEED, SUICIDAL, SKIP, WARNING;
private static Multimap<ExecutableState, ExecutableState> VALID_STATE_TRANSFER;
@@ -60,6 +60,7 @@ public enum ExecutableState {
VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.SUICIDAL);
VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.PAUSED);
VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.SKIP);
+ VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.WARNING);
VALID_STATE_TRANSFER.put(ExecutableState.PAUSED, ExecutableState.DISCARDED);
VALID_STATE_TRANSFER.put(ExecutableState.PAUSED, ExecutableState.SUICIDAL);
@@ -94,6 +95,12 @@ public enum ExecutableState {
this == READY;//restart case
}
+ public boolean isNotBad() {
+ return this == SUCCEED
+ || this == SKIP
+ || this == WARNING;
+ }
+
public static boolean isValidStateTransfer(ExecutableState from, ExecutableState to) {
return VALID_STATE_TRANSFER.containsEntry(from, to);
}
@@ -115,6 +122,8 @@ public enum ExecutableState {
case SUICIDAL:
case DISCARDED:
return JobStatusEnum.DISCARDED;
+ case WARNING:
+ return JobStatusEnum.WARNING;
default:
throw new RuntimeException("invalid state:" + this);
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
index c9567f1b10..86c60b35dc 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecuteResult.java
@@ -47,17 +47,18 @@ public final class ExecuteResult {
private ExecuteResult(State state, String output, Throwable throwable) {
Preconditions.checkArgument(state != null, "state cannot be null");
- if (state == State.SUCCEED) {
- Preconditions.checkNotNull(output);
- Preconditions.checkState(throwable == null);
- } else if (state == State.SKIP) {
- Preconditions.checkNotNull(output);
- Preconditions.checkState(throwable == null);
- } else if (state == State.ERROR) {
- Preconditions.checkNotNull(throwable);
- Preconditions.checkState(output == null);
- } else {
- throw new IllegalStateException();
+ switch (state) {
+ case SUCCEED:
+ case SKIP:
+ Preconditions.checkNotNull(output);
+ Preconditions.checkState(throwable == null);
+ break;
+ case ERROR:
+ Preconditions.checkNotNull(throwable);
+ Preconditions.checkState(output == null);
+ break;
+ default:
+ throw new IllegalStateException();
}
this.state = state;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
index b22dee724f..4f1b192f1d 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
@@ -1238,6 +1238,7 @@ public class NExecutableManager {
// DISCARDED must not be transferred to any others status
if ((oldStatus == ExecutableState.PAUSED && newStatus == ExecutableState.ERROR)
|| (oldStatus == ExecutableState.SKIP && newStatus == ExecutableState.SUCCEED)
+ || (oldStatus == ExecutableState.WARNING && newStatus == ExecutableState.SUCCEED)
|| oldStatus == ExecutableState.DISCARDED) {
return false;
}
@@ -1253,6 +1254,12 @@ public class NExecutableManager {
final int indexSuccessCount = Integer
.parseInt(map.getOrDefault(NBatchConstants.P_INDEX_SUCCESS_COUNT, "0"));
info.put(NBatchConstants.P_INDEX_SUCCESS_COUNT, String.valueOf(indexSuccessCount));
+
+ // Add warning_code to stage output info if exists
+ String warningCode;
+ if ((warningCode = map.get(NBatchConstants.P_WARNING_CODE)) != null) {
+ info.put(NBatchConstants.P_WARNING_CODE, warningCode);
+ }
});
jobOutput.setInfo(info);
jobOutput.setLastModified(System.currentTimeMillis());
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java b/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
index 4ba31b27c1..3f05c47032 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
@@ -162,6 +162,27 @@ public class NExecutableManagerTest extends NLocalFileMetadataTestCase {
assertJobEqual(job, anotherJob);
}
+ @Test
+ public void testExecutableStateCorrectness() {
+ assertTrue(ExecutableState.READY.isProgressing());
+ assertTrue(ExecutableState.RUNNING.isProgressing());
+
+ assertTrue(ExecutableState.SUCCEED.isFinalState());
+ assertTrue(ExecutableState.DISCARDED.isFinalState());
+ assertTrue(ExecutableState.SUICIDAL.isFinalState());
+
+ assertTrue(ExecutableState.ERROR.isNotProgressing());
+ assertTrue(ExecutableState.PAUSED.isNotProgressing());
+
+ assertTrue(ExecutableState.DISCARDED.isStoppedNonVoluntarily());
+ assertTrue(ExecutableState.PAUSED.isStoppedNonVoluntarily());
+ assertTrue(ExecutableState.READY.isStoppedNonVoluntarily());
+
+ assertTrue(ExecutableState.SUCCEED.isNotBad());
+ assertTrue(ExecutableState.SKIP.isNotBad());
+ assertTrue(ExecutableState.WARNING.isNotBad());
+ }
+
@Test
public void testValidStateTransfer() {
SucceedTestExecutable job = new SucceedTestExecutable();
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NBatchConstants.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NBatchConstants.java
index 754769280b..bcca549182 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NBatchConstants.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NBatchConstants.java
@@ -55,6 +55,7 @@ public interface NBatchConstants {
/** use for stage calculate exec ratio */
String P_INDEX_COUNT = "indexCount";
String P_INDEX_SUCCESS_COUNT = "indexSuccessCount";
+ String P_WARNING_CODE = "warning_code";
/** value like : { "segmentId1": 1223, "segmentId2": 1223 } */
String P_WAITE_TIME = "waiteTime";
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLayout.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLayout.java
index d979ece2f8..08a99459e1 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLayout.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLayout.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -43,6 +44,10 @@ import lombok.val;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class NDataLayout implements Serializable {
+ public enum AbnormalType {
+ DATA_INCONSISTENT
+ }
+
public static NDataLayout newDataLayout(NDataflow df, String segId, long layoutId) {
return newDataLayout(NDataSegDetails.newSegDetails(df, segId), layoutId);
}
@@ -54,6 +59,13 @@ public class NDataLayout implements Serializable {
return r;
}
+ public static boolean filterWorkingLayout(NDataLayout layout) {
+ if (layout == null) {
+ return false;
+ }
+ return Objects.isNull(layout.getAbnormalType());
+ }
+
// ============================================================================
/**
@@ -97,6 +109,11 @@ public class NDataLayout implements Serializable {
@JsonProperty("multi_partition")
private List<LayoutPartition> multiPartition = new ArrayList<>();
+ @Getter
+ @Setter
+ @JsonProperty("abnormal_type")
+ private AbnormalType abnormalType;
+
public NDataLayout() {
this.createTime = System.currentTimeMillis();
}
@@ -260,7 +277,7 @@ public class NDataLayout implements Serializable {
if (segDetails == null || !segDetails.isCachedAndShared())
return false;
- for (NDataLayout cached : segDetails.getLayouts()) {
+ for (NDataLayout cached : segDetails.getWorkingLayouts()) {
if (cached == this)
return true;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetails.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetails.java
index 5607e918c5..e9866f3509 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetails.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetails.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.cube.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.RootPersistentEntity;
@@ -114,18 +115,40 @@ public class NDataSegDetails extends RootPersistentEntity implements Serializabl
public long getTotalRowCount() {
long count = 0L;
- for (NDataLayout cuboid : getLayouts()) {
+ for (NDataLayout cuboid : getWorkingLayouts()) {
count += cuboid.getRows();
}
return count;
}
+ /**
+ * @deprecated Deprecated because of non-working layouts were added.
+ * <p>Use {@link NDataSegDetails#getWorkingLayouts} or {@link NDataSegDetails#getAllLayouts} instead.
+ */
+ @Deprecated
public List<NDataLayout> getLayouts() {
- return isCachedAndShared() ? ImmutableList.copyOf(layouts) : layouts;
+ return getAllLayouts();
+ }
+
+ public List<NDataLayout> getWorkingLayouts() {
+ List<NDataLayout> workingLayouts = getLayouts0(false);
+ return isCachedAndShared() ? ImmutableList.copyOf(workingLayouts) : workingLayouts;
+ }
+
+ public List<NDataLayout> getAllLayouts() {
+ List<NDataLayout> allLayouts = getLayouts0(true);
+ return isCachedAndShared() ? ImmutableList.copyOf(allLayouts) : allLayouts;
+ }
+
+ private List<NDataLayout> getLayouts0(boolean includingNonWorkingLayouts) {
+ if (includingNonWorkingLayouts) {
+ return layouts;
+ }
+ return layouts.stream().filter(NDataLayout::filterWorkingLayout).collect(Collectors.toList());
}
public NDataLayout getLayoutById(long layoutId) {
- for (NDataLayout cuboid : getLayouts()) {
+ for (NDataLayout cuboid : getAllLayouts()) {
if (cuboid.getLayoutId() == layoutId)
return cuboid;
}
@@ -156,8 +179,8 @@ public class NDataSegDetails extends RootPersistentEntity implements Serializabl
if (another == this)
return false;
- List<NDataLayout> currentSortedLayouts = getSortedLayouts(getLayouts());
- List<NDataLayout> anotherSortedLayouts = getSortedLayouts(another.getLayouts());
+ List<NDataLayout> currentSortedLayouts = getSortedLayouts(getAllLayouts());
+ List<NDataLayout> anotherSortedLayouts = getSortedLayouts(another.getAllLayouts());
int size = currentSortedLayouts.size();
if (size != anotherSortedLayouts.size())
return false;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
index ddb81e407a..00c2d105a7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
@@ -275,7 +275,20 @@ public class NDataSegment implements ISegment, Serializable {
return getLayoutInfo().getLayoutSize();
}
+ @Override
+ public int getWorkingLayoutSize() {
+ return (int) getLayoutInfo().getLayoutsMap().values().stream()
+ .filter(NDataLayout::filterWorkingLayout).count();
+ }
+
public NDataLayout getLayout(long layoutId) {
+ return getLayout(layoutId, false);
+ }
+
+ public NDataLayout getLayout(long layoutId, boolean includingNonWorkingLayout) {
+ if (includingNonWorkingLayout) {
+ return getAllLayoutsMap().get(layoutId);
+ }
return getLayoutsMap().get(layoutId);
}
@@ -283,6 +296,10 @@ public class NDataSegment implements ISegment, Serializable {
return getLayoutInfo().getLayoutsMap();
}
+ public Map<Long, NDataLayout> getAllLayoutsMap() {
+ return getLayoutInfo().getAllLayoutsMap();
+ }
+
public Set<Long> getLayoutIds() {
return getLayoutInfo().getLayoutIds();
}
@@ -314,7 +331,8 @@ public class NDataSegment implements ISegment, Serializable {
// not required by spark cubing
private NDataSegDetails segDetails;
// not required by spark cubing
- private Map<Long, NDataLayout> layoutsMap = Collections.emptyMap();
+ private Map<Long, NDataLayout> allLayoutsMap = Collections.emptyMap();
+ private Map<Long, NDataLayout> workingLayoutsMap = Collections.emptyMap();
/**
* for each layout, partition id -> bucket id
*/
@@ -329,7 +347,10 @@ public class NDataSegment implements ISegment, Serializable {
}
public LayoutInfo(Map<Long, NDataLayout> layoutsMap) {
- this.layoutsMap = layoutsMap;
+ this.allLayoutsMap = layoutsMap;
+ this.workingLayoutsMap = layoutsMap.entrySet().stream()
+ .filter(entry -> NDataLayout.filterWorkingLayout(entry.getValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
private LayoutInfo(boolean loadDetail) {
@@ -344,37 +365,45 @@ public class NDataSegment implements ISegment, Serializable {
IndexPlan indexPlan = dataflow.getIndexPlan();
if (!indexPlan.isBroken()) {
- List<NDataLayout> filteredCuboids = segDetails.getLayouts().stream()
+ List<NDataLayout> filteredCuboids = segDetails.getAllLayouts().stream()
.filter(dataLayout -> dataLayout.getLayout() != null).collect(Collectors.toList());
segDetails.setLayouts(filteredCuboids);
}
segDetails.setCachedAndShared(dataflow.isCachedAndShared());
- List<NDataLayout> cuboids = segDetails.getLayouts();
- layoutsMap = new HashMap<>(cuboids.size());
- for (NDataLayout cuboid : cuboids) {
- layoutsMap.put(cuboid.getLayoutId(), cuboid);
- Map<Long, Long> cuboidBucketMap = Maps.newHashMap();
- cuboid.getMultiPartition().forEach(dataPartition -> cuboidBucketMap.put(dataPartition.getPartitionId(),
- dataPartition.getBucketId()));
- partitionBucketMap.put(cuboid.getLayoutId(), cuboidBucketMap);
+ List<NDataLayout> allLayouts = segDetails.getAllLayouts();
+ allLayoutsMap = Maps.newHashMap();
+ workingLayoutsMap = Maps.newHashMap();
+ for (NDataLayout layout : allLayouts) {
+ allLayoutsMap.put(layout.getLayoutId(), layout);
+ if (NDataLayout.filterWorkingLayout(layout)) {
+ workingLayoutsMap.put(layout.getLayoutId(), layout);
+ Map<Long, Long> cuboidBucketMap = Maps.newHashMap();
+ layout.getMultiPartition().forEach(dataPartition -> cuboidBucketMap.put(dataPartition.getPartitionId(),
+ dataPartition.getBucketId()));
+ partitionBucketMap.put(layout.getLayoutId(), cuboidBucketMap);
+ }
}
}
public int getLayoutSize() {
- return layoutsMap.size();
+ return workingLayoutsMap.size();
}
public NDataLayout getLayout(long layoutId) {
- return layoutsMap.get(layoutId);
+ return workingLayoutsMap.get(layoutId);
}
public Map<Long, NDataLayout> getLayoutsMap() {
- return layoutsMap;
+ return workingLayoutsMap;
+ }
+
+ public Map<Long, NDataLayout> getAllLayoutsMap() {
+ return allLayoutsMap;
}
public Set<Long> getLayoutIds() {
- return layoutsMap.keySet();
+ return workingLayoutsMap.keySet();
}
public List<Long> getMultiPartitionIds() {
@@ -390,8 +419,8 @@ public class NDataSegment implements ISegment, Serializable {
}
public boolean isAlreadyBuilt(long layoutId) {
- if (Objects.nonNull(layoutsMap) && layoutsMap.containsKey(layoutId)) {
- return layoutsMap.get(layoutId).isReady();
+ if (Objects.nonNull(workingLayoutsMap) && workingLayoutsMap.containsKey(layoutId)) {
+ return workingLayoutsMap.get(layoutId).isReady();
}
return false;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
index f22dd1838d..875201fe65 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
@@ -730,7 +730,7 @@ public class NDataflowManager implements IRealizationProvider {
private void updateSegmentStatus(NDataSegment seg) {
NDataSegDetails segDetails = NDataSegDetailsManager.getInstance(seg.getConfig(), project).getForSegment(seg);
- if (seg.getStatus() == SegmentStatusEnum.WARNING && segDetails != null && segDetails.getLayouts().isEmpty()) {
+ if (seg.getStatus() == SegmentStatusEnum.WARNING && segDetails != null && segDetails.getAllLayouts().isEmpty()) {
seg.setStatus(SegmentStatusEnum.READY);
}
}
@@ -830,7 +830,7 @@ public class NDataflowManager implements IRealizationProvider {
}
val affectedLayouts = Lists.newArrayList();
for (NDataSegment segment : updateSegments) {
- val layouts = segment.getSegDetails().getLayouts();
+ val layouts = segment.getSegDetails().getAllLayouts();
layouts.forEach(dataLayout -> {
if (dataLayout.removeMultiPartition(toBeDeletedPartIds)) {
affectedLayouts.add(dataLayout);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/SegmentPartition.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/SegmentPartition.java
index dfe2cd16f9..0b4b8ca873 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/SegmentPartition.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/SegmentPartition.java
@@ -142,7 +142,7 @@ public class SegmentPartition implements Serializable {
return 0;
}
storageSize = dataSegment.getSegDetails() //
- .getLayouts().stream() //
+ .getWorkingLayouts().stream() //
.flatMap(layout -> layout.getMultiPartition().stream()) //
.filter(partition -> partition.getPartitionId() == partitionId) //
.mapToLong(LayoutPartition::getByteSize).sum();
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index 46f72933ef..146eead0f4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -43,6 +43,8 @@ public interface ISegment extends Comparable<ISegment> {
public int getLayoutSize();
+ public int getWorkingLayoutSize();
+
public NDataModel getModel();
public SegmentStatusEnum getStatus();
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
index 0d2e6cbaf3..f3ed44b83f 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
@@ -68,7 +68,7 @@ public class NProjectManagerTest extends NLocalFileMetadataTestCase {
}
val projects = projectManager.listAllProjects();
- Assert.assertEquals(28, projects.size());
+ Assert.assertEquals(29, projects.size());
Assert.assertTrue(projects.stream().noneMatch(p -> p.getName().equals("test")));
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
index ce18241853..28a8c5fada 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
@@ -90,7 +90,7 @@ public class HdfsCapacityMetricsTest extends NLocalFileMetadataTestCase {
}
Assert.assertTrue(hdfsCapacityMetrics.getWorkingDirCapacity().isEmpty());
hdfsCapacityMetrics.writeHdfsMetrics();
- Assert.assertEquals(28, hdfsCapacityMetrics.getWorkingDirCapacity().size());
+ Assert.assertEquals(29, hdfsCapacityMetrics.getWorkingDirCapacity().size());
}
@@ -139,4 +139,4 @@ public class HdfsCapacityMetricsTest extends NLocalFileMetadataTestCase {
HdfsCapacityMetrics hdfsCapacityMetrics = new HdfsCapacityMetrics(getTestConfig());
Assert.assertEquals(0L, (long) hdfsCapacityMetrics.getHdfsCapacityByProject("kylin"));
}
-}
\ No newline at end of file
+}
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
index c974b4caeb..36719e6abb 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
@@ -249,7 +249,8 @@ public class ExecutableResponse implements Comparable<ExecutableResponse> {
continue;
}
}
- if (ExecutableState.SUCCEED == task.getStatus() || ExecutableState.SKIP == task.getStatus()) {
+
+ if (task.getStatus().isNotBad()) {
successSteps++;
}
}
@@ -279,7 +280,8 @@ public class ExecutableResponse implements Comparable<ExecutableResponse> {
var successStages = 0D;
for (StageBase stage : stageBases) {
if (ExecutableState.SUCCEED == stage.getStatus(segmentId)
- || stage.getStatus(segmentId) == ExecutableState.SKIP) {
+ || ExecutableState.SKIP == stage.getStatus(segmentId)
+ || ExecutableState.WARNING == stage.getStatus(segmentId)) {
successStages += 1;
continue;
}
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableStepResponse.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableStepResponse.java
index edd980320f..c0c9f1282b 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableStepResponse.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableStepResponse.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.collect.Maps;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.constant.JobStepCmdTypeEnum;
@@ -152,5 +153,7 @@ public class ExecutableStepResponse {
private long execEndTime;
@JsonProperty("stage")
private List<ExecutableStepResponse> stage;
+ @JsonProperty("info")
+ private Map<String, String> info = Maps.newHashMap();
}
}
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index eb668435fa..dad56e01d3 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -829,6 +829,12 @@ public class JobService extends BasicService implements JobSupporter, ISmartAppl
val stepCount = stageResponses.isEmpty() ? 1 : stageResponses.size();
val stepRatio = (float) ExecutableResponse.calculateSuccessStage(task, segmentId, stageBases, true) / stepCount;
segmentSubStages.setStepRatio(stepRatio);
+
+ // Put warning message into segment_sub_stages.info if exists
+ Optional<ExecutableStepResponse> warningStageRes = stageResponses.stream().filter(stageRes ->
+ stageRes.getStatus() == JobStatusEnum.WARNING).findFirst();
+ warningStageRes.ifPresent(res -> segmentSubStages.getInfo().put(NBatchConstants.P_WARNING_CODE,
+ res.getInfo().getOrDefault(NBatchConstants.P_WARNING_CODE, null)));
}
private void setStage(List<ExecutableStepResponse> responses, ExecutableStepResponse newResponse) {
@@ -844,7 +850,7 @@ public class JobService extends BasicService implements JobSupporter, ISmartAppl
*/
Set<JobStatusEnum> jobStatusEnums = Sets.newHashSet(JobStatusEnum.ERROR, JobStatusEnum.STOPPED,
JobStatusEnum.DISCARDED);
- Set<JobStatusEnum> jobFinishOrSkip = Sets.newHashSet(JobStatusEnum.FINISHED, JobStatusEnum.SKIP);
+ Set<JobStatusEnum> jobFinishOrSkip = Sets.newHashSet(JobStatusEnum.FINISHED, JobStatusEnum.SKIP, JobStatusEnum.WARNING);
if (oldResponse.getStatus() != newResponse.getStatus()
&& !jobStatusEnums.contains(oldResponse.getStatus())) {
if (jobStatusEnums.contains(newResponse.getStatus())) {
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/StageTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/StageTest.java
index b69c00eb9b..e565a194fd 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/StageTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/StageTest.java
@@ -549,6 +549,9 @@ public class StageTest extends NLocalFileMetadataTestCase {
result = ExecutableState.DISCARDED.toJobStatus();
Assert.assertEquals(JobStatusEnum.DISCARDED, result);
+
+ result = ExecutableState.WARNING.toJobStatus();
+ Assert.assertEquals(JobStatusEnum.WARNING, result);
}
@Test
diff --git a/src/examples/test_case_data/localmeta/metadata/_global/project/index_build_test.json b/src/examples/test_case_data/localmeta/metadata/_global/project/index_build_test.json
new file mode 100644
index 0000000000..7c1aadeca6
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/_global/project/index_build_test.json
@@ -0,0 +1,40 @@
+{
+ "uuid": "63b21fc0-1bbd-3570-b2ac-54fbb0fb8181",
+ "last_modified": 1675944232432,
+ "create_time": 1675944228905,
+ "version": "4.0.0.0",
+ "name": "index_build_test",
+ "owner": "ADMIN",
+ "status": "ENABLED",
+ "create_time_utc": 1675944228905,
+ "default_database": "DEFAULT",
+ "description": null,
+ "principal": null,
+ "keytab": null,
+ "maintain_model_type": "MANUAL_MAINTAIN",
+ "override_kylin_properties": {
+ "kylin.metadata.semi-automatic-mode": "true",
+ "kylin.query.metadata.expose-computed-column": "true",
+ "kylin.source.default": "9"
+ },
+ "segment_config": {
+ "auto_merge_enabled": false,
+ "auto_merge_time_ranges": [
+ "WEEK",
+ "MONTH",
+ "QUARTER",
+ "YEAR"
+ ],
+ "volatile_range": {
+ "volatile_range_number": 0,
+ "volatile_range_enabled": false,
+ "volatile_range_type": "DAY"
+ },
+ "retention_range": {
+ "retention_range_number": 1,
+ "retention_range_enabled": false,
+ "retention_range_type": "MONTH"
+ },
+ "create_empty_segment_enabled": false
+ }
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow/3ec47efc-573a-9304-4405-8e05ae184322.json b/src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow/3ec47efc-573a-9304-4405-8e05ae184322.json
new file mode 100644
index 0000000000..d020817c44
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow/3ec47efc-573a-9304-4405-8e05ae184322.json
@@ -0,0 +1,70 @@
+{
+ "uuid": "3ec47efc-573a-9304-4405-8e05ae184322",
+ "last_modified": 1675944717956,
+ "create_time": 1675944415945,
+ "version": "4.0.0.0",
+ "status": "ONLINE",
+ "last_status": null,
+ "cost": 50,
+ "query_hit_count": 0,
+ "last_query_time": 0,
+ "layout_query_hit_count": {},
+ "segments": [
+ {
+ "id": "b2f206e1-7a15-c94a-20f5-f608d550ead6",
+ "name": "FULL_BUILD",
+ "create_time_utc": 1675944504890,
+ "status": "READY",
+ "segRange": {
+ "@class": "org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start": 0,
+ "date_range_end": 9223372036854775807
+ },
+ "timeRange": null,
+ "dimension_range_info_map": {
+ "0": {
+ "min": "1",
+ "max": "6"
+ },
+ "24": {
+ "min": "1",
+ "max": "5"
+ },
+ "13": {
+ "min": "5",
+ "max": "15"
+ },
+ "22": {
+ "min": "Customer#000000001",
+ "max": "Customer#000000005"
+ },
+ "16": {
+ "min": "1",
+ "max": "5"
+ }
+ },
+ "parameters": null,
+ "dictionaries": null,
+ "snapshots": null,
+ "last_build_time": 1675944668815,
+ "source_count": 6,
+ "source_bytes_size": 27732,
+ "column_source_bytes": {
+ "SSB.CUSTOMER.C_NAME": 108,
+ "SSB.LINEORDER.LO_QUANTITY": 9,
+ "SSB.LINEORDER.LO_ORDERKEY": 6,
+ "SSB.LINEORDER.LO_CUSTKEY": 6,
+ "SSB.CUSTOMER.C_CUSTKEY": 6
+ },
+ "ori_snapshot_size": {},
+ "additionalInfo": {},
+ "is_realtime_segment": false,
+ "is_snapshot_ready": false,
+ "is_dict_ready": true,
+ "is_flat_table_ready": true,
+ "is_fact_view_ready": false,
+ "multi_partitions": [],
+ "max_bucket_id": -1
+ }
+ ]
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow_details/3ec47efc-573a-9304-4405-8e05ae184322/b2f206e1-7a15-c94a-20f5-f608d550ead6.json b/src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow_details/3ec47efc-573a-9304-4405-8e05ae184322/b2f206e1-7a15-c94a-20f5-f608d550ead6.json
new file mode 100644
index 0000000000..b7512dc9f2
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/index_build_test/dataflow_details/3ec47efc-573a-9304-4405-8e05ae184322/b2f206e1-7a15-c94a-20f5-f608d550ead6.json
@@ -0,0 +1,69 @@
+{
+ "uuid": "b2f206e1-7a15-c94a-20f5-f608d550ead6",
+ "last_modified": 1675944504890,
+ "create_time": 1675944504890,
+ "version": "4.0.0.0",
+ "dataflow": "3ec47efc-573a-9304-4405-8e05ae184322",
+ "layout_instances": [
+ {
+ "layout_id": 1,
+ "build_job_id": "944d7367-29e5-a5bd-373c-f01b34c078d2-3ec47efc-573a-9304-4405-8e05ae184322",
+ "rows": 6,
+ "byte_size": 2021,
+ "file_count": 1,
+ "source_rows": 6,
+ "source_byte_size": 0,
+ "partition_num": 1,
+ "partition_values": [],
+ "is_ready": false,
+ "create_time": 1675944665659,
+ "multi_partition": [],
+ "abnormal_type": null
+ },
+ {
+ "layout_id": 10001,
+ "build_job_id": "944d7367-29e5-a5bd-373c-f01b34c078d2-3ec47efc-573a-9304-4405-8e05ae184322",
+ "rows": 6,
+ "byte_size": 2021,
+ "file_count": 1,
+ "source_rows": 6,
+ "source_byte_size": 0,
+ "partition_num": 1,
+ "partition_values": [],
+ "is_ready": false,
+ "create_time": 1675944665659,
+ "multi_partition": [],
+ "abnormal_type": null
+ },
+ {
+ "layout_id": 20000000001,
+ "build_job_id": "944d7367-29e5-a5bd-373c-f01b34c078d2-3ec47efc-573a-9304-4405-8e05ae184322",
+ "rows": 6,
+ "byte_size": 1739,
+ "file_count": 1,
+ "source_rows": 6,
+ "source_byte_size": 0,
+ "partition_num": 1,
+ "partition_values": [],
+ "is_ready": false,
+ "create_time": 1675944665659,
+ "multi_partition": [],
+ "abnormal_type": null
+ },
+ {
+ "layout_id": 20000010001,
+ "build_job_id": "944d7367-29e5-a5bd-373c-f01b34c078d2-3ec47efc-573a-9304-4405-8e05ae184322",
+ "rows": 6,
+ "byte_size": 1739,
+ "file_count": 1,
+ "source_rows": 6,
+ "source_byte_size": 0,
+ "partition_num": 1,
+ "partition_values": [],
+ "is_ready": false,
+ "create_time": 1675944665659,
+ "multi_partition": [],
+ "abnormal_type": null
+ }
+ ]
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/index_build_test/index_plan/3ec47efc-573a-9304-4405-8e05ae184322.json b/src/examples/test_case_data/localmeta/metadata/index_build_test/index_plan/3ec47efc-573a-9304-4405-8e05ae184322.json
new file mode 100644
index 0000000000..f53bfcb30c
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/index_build_test/index_plan/3ec47efc-573a-9304-4405-8e05ae184322.json
@@ -0,0 +1,204 @@
+{
+ "uuid": "3ec47efc-573a-9304-4405-8e05ae184322",
+ "last_modified": 1675945219064,
+ "create_time": 1675944415943,
+ "version": "4.0.0.0",
+ "description": null,
+ "rule_based_index": null,
+ "indexes": [
+ {
+ "id": 10000,
+ "dimensions": [
+ 24,
+ 22
+ ],
+ "measures": [
+ 100000,
+ 100001
+ ],
+ "layouts": [
+ {
+ "id": 10001,
+ "name": null,
+ "owner": null,
+ "col_order": [
+ 24,
+ 22,
+ 100000,
+ 100001
+ ],
+ "shard_by_columns": [],
+ "partition_by_columns": [],
+ "sort_by_columns": [],
+ "storage_type": 20,
+ "update_time": 1675945215262,
+ "manual": false,
+ "auto": true,
+ "base": false,
+ "draft_version": null,
+ "index_range": null
+ }
+ ],
+ "next_layout_offset": 2
+ },
+ {
+ "id": 20000000000,
+ "dimensions": [
+ 0,
+ 13,
+ 16,
+ 24,
+ 22
+ ],
+ "measures": [],
+ "layouts": [
+ {
+ "id": 20000000001,
+ "name": null,
+ "owner": null,
+ "col_order": [
+ 0,
+ 13,
+ 16,
+ 24,
+ 22
+ ],
+ "shard_by_columns": [],
+ "partition_by_columns": [],
+ "sort_by_columns": [],
+ "storage_type": 20,
+ "update_time": 1675944415943,
+ "manual": false,
+ "auto": false,
+ "base": true,
+ "draft_version": null,
+ "index_range": null
+ }
+ ],
+ "next_layout_offset": 2
+ },
+ {
+ "id": 20000010000,
+ "dimensions": [
+ 0,
+ 13
+ ],
+ "measures": [],
+ "layouts": [
+ {
+ "id": 20000010001,
+ "name": null,
+ "owner": null,
+ "col_order": [
+ 0,
+ 13
+ ],
+ "shard_by_columns": [],
+ "partition_by_columns": [],
+ "sort_by_columns": [],
+ "storage_type": 20,
+ "update_time": 1675944415943,
+ "manual": false,
+ "auto": false,
+ "base": true,
+ "draft_version": null,
+ "index_range": null
+ }
+ ],
+ "next_layout_offset": 2
+ },
+ {
+ "id": 20000,
+ "dimensions": [
+ 0,
+ 13,
+ 16,
+ 24,
+ 22
+ ],
+ "measures": [
+ 100000,
+ 100001
+ ],
+ "layouts": [
+ {
+ "id": 20001,
+ "name": null,
+ "owner": "ADMIN",
+ "col_order": [
+ 0,
+ 13,
+ 16,
+ 24,
+ 22,
+ 100000,
+ 100001
+ ],
+ "shard_by_columns": [],
+ "partition_by_columns": [],
+ "sort_by_columns": [],
+ "storage_type": 20,
+ "update_time": 1675945219064,
+ "manual": false,
+ "auto": false,
+ "base": true,
+ "draft_version": null,
+ "index_range": null
+ }
+ ],
+ "next_layout_offset": 2
+ }
+ ],
+ "override_properties": {},
+ "to_be_deleted_indexes": [
+ {
+ "id": 0,
+ "dimensions": [
+ 0,
+ 13,
+ 16,
+ 24,
+ 22
+ ],
+ "measures": [
+ 100000
+ ],
+ "layouts": [
+ {
+ "id": 1,
+ "name": null,
+ "owner": null,
+ "col_order": [
+ 0,
+ 13,
+ 16,
+ 24,
+ 22,
+ 100000
+ ],
+ "shard_by_columns": [],
+ "partition_by_columns": [],
+ "sort_by_columns": [],
+ "storage_type": 20,
+ "update_time": 1675944415943,
+ "manual": false,
+ "auto": false,
+ "base": true,
+ "draft_version": null,
+ "index_range": null
+ }
+ ],
+ "next_layout_offset": 2
+ }
+ ],
+ "auto_merge_time_ranges": null,
+ "retention_range": 0,
+ "engine_type": 80,
+ "next_aggregation_index_id": 30000,
+ "next_table_index_id": 20000020000,
+ "agg_shard_by_columns": [],
+ "extend_partition_columns": [],
+ "layout_bucket_num": {},
+ "approved_additional_recs": 1,
+ "approved_removal_recs": 0
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/index_build_test/model_desc/3ec47efc-573a-9304-4405-8e05ae184322.json b/src/examples/test_case_data/localmeta/metadata/index_build_test/model_desc/3ec47efc-573a-9304-4405-8e05ae184322.json
new file mode 100644
index 0000000000..f75ddbffb2
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/index_build_test/model_desc/3ec47efc-573a-9304-4405-8e05ae184322.json
@@ -0,0 +1,241 @@
+{
+ "uuid": "3ec47efc-573a-9304-4405-8e05ae184322",
+ "last_modified": 1675999084906,
+ "create_time": 1675999084809,
+ "version": "4.0.0.0",
+ "alias": "INDEX_BUILD_ON_SEGMENT",
+ "owner": "ADMIN",
+ "config_last_modifier": null,
+ "config_last_modified": 0,
+ "description": "",
+ "fact_table": "SSB.LINEORDER",
+ "fact_table_alias": null,
+ "management_type": "MODEL_BASED",
+ "join_tables": [
+ {
+ "table": "SSB.CUSTOMER",
+ "kind": "LOOKUP",
+ "alias": "CUSTOMER",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "CUSTOMER.C_CUSTKEY"
+ ],
+ "foreign_key": [
+ "LINEORDER.LO_CUSTKEY"
+ ],
+ "non_equi_join_condition": null,
+ "primary_table": null,
+ "foreign_table": null
+ },
+ "flattenable": "flatten",
+ "join_relation_type": "MANY_TO_ONE"
+ }
+ ],
+ "filter_condition": "",
+ "partition_desc": null,
+ "capacity": "MEDIUM",
+ "segment_config": {
+ "auto_merge_enabled": null,
+ "auto_merge_time_ranges": null,
+ "volatile_range": null,
+ "retention_range": null,
+ "create_empty_segment_enabled": false
+ },
+ "data_check_desc": null,
+ "semantic_version": 0,
+ "storage_type": 0,
+ "model_type": "BATCH",
+ "all_named_columns": [
+ {
+ "id": 0,
+ "name": "LO_ORDERKEY",
+ "column": "LINEORDER.LO_ORDERKEY",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 1,
+ "name": "LO_PARTKEY",
+ "column": "LINEORDER.LO_PARTKEY"
+ },
+ {
+ "id": 2,
+ "name": "LO_DISCOUNT",
+ "column": "LINEORDER.LO_DISCOUNT"
+ },
+ {
+ "id": 3,
+ "name": "LO_SUPPLYCOST",
+ "column": "LINEORDER.LO_SUPPLYCOST"
+ },
+ {
+ "id": 4,
+ "name": "LO_COMMITDATE",
+ "column": "LINEORDER.LO_COMMITDATE"
+ },
+ {
+ "id": 5,
+ "name": "LO_EXTENDEDPRICE",
+ "column": "LINEORDER.LO_EXTENDEDPRICE"
+ },
+ {
+ "id": 6,
+ "name": "LO_TAX",
+ "column": "LINEORDER.LO_TAX"
+ },
+ {
+ "id": 7,
+ "name": "LO_SUPPKEY",
+ "column": "LINEORDER.LO_SUPPKEY"
+ },
+ {
+ "id": 8,
+ "name": "LO_ORDTOTALPRICE",
+ "column": "LINEORDER.LO_ORDTOTALPRICE"
+ },
+ {
+ "id": 9,
+ "name": "LO_REVENUE",
+ "column": "LINEORDER.LO_REVENUE"
+ },
+ {
+ "id": 10,
+ "name": "LO_ORDERDATE",
+ "column": "LINEORDER.LO_ORDERDATE"
+ },
+ {
+ "id": 11,
+ "name": "LO_ORDERPRIOTITY",
+ "column": "LINEORDER.LO_ORDERPRIOTITY"
+ },
+ {
+ "id": 12,
+ "name": "LO_SHIPPRIOTITY",
+ "column": "LINEORDER.LO_SHIPPRIOTITY"
+ },
+ {
+ "id": 13,
+ "name": "LO_QUANTITY",
+ "column": "LINEORDER.LO_QUANTITY",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 14,
+ "name": "LO_SHIPMODE",
+ "column": "LINEORDER.LO_SHIPMODE"
+ },
+ {
+ "id": 15,
+ "name": "LO_LINENUMBER",
+ "column": "LINEORDER.LO_LINENUMBER"
+ },
+ {
+ "id": 16,
+ "name": "LO_CUSTKEY",
+ "column": "LINEORDER.LO_CUSTKEY",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 17,
+ "name": "C_ADDRESS",
+ "column": "CUSTOMER.C_ADDRESS"
+ },
+ {
+ "id": 18,
+ "name": "C_NATION",
+ "column": "CUSTOMER.C_NATION"
+ },
+ {
+ "id": 19,
+ "name": "C_CITY",
+ "column": "CUSTOMER.C_CITY"
+ },
+ {
+ "id": 20,
+ "name": "C_PHONE",
+ "column": "CUSTOMER.C_PHONE"
+ },
+ {
+ "id": 21,
+ "name": "C_REGION",
+ "column": "CUSTOMER.C_REGION"
+ },
+ {
+ "id": 22,
+ "name": "C_NAME",
+ "column": "CUSTOMER.C_NAME",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 23,
+ "name": "C_MKTSEGMENT",
+ "column": "CUSTOMER.C_MKTSEGMENT"
+ },
+ {
+ "id": 24,
+ "name": "C_CUSTKEY",
+ "column": "CUSTOMER.C_CUSTKEY",
+ "status": "DIMENSION"
+ }
+ ],
+ "all_measures": [
+ {
+ "name": "COUNT_ALL",
+ "function": {
+ "expression": "COUNT",
+ "parameters": [
+ {
+ "type": "constant",
+ "value": "1"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "column": null,
+ "comment": null,
+ "id": 100000,
+ "type": "NORMAL",
+ "internal_ids": []
+ },
+ {
+ "name": "SUM_LINEORDER_LO_QUANTITY",
+ "function": {
+ "expression": "SUM",
+ "parameters": [
+ {
+ "type": "column",
+ "value": "LINEORDER.LO_QUANTITY"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "column": null,
+ "comment": null,
+ "id": 100001,
+ "type": "NORMAL",
+ "internal_ids": []
+ }
+ ],
+ "recommendations_count": 0,
+ "computed_columns": [],
+ "canvas": {
+ "coordinate": {
+ "LINEORDER": {
+ "x": 752.8888617621528,
+ "y": 126.22219509548611,
+ "width": 200.0,
+ "height": 230.0
+ },
+ "CUSTOMER": {
+ "x": 745.111083984375,
+ "y": 468.44441731770837,
+ "width": 200.0,
+ "height": 230.0
+ }
+ },
+ "zoom": 9.0
+ },
+ "multi_partition_desc": null,
+ "multi_partition_key_mapping": null,
+ "fusion_id": null
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.CUSTOMER.json b/src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.CUSTOMER.json
new file mode 100644
index 0000000000..c84432550e
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.CUSTOMER.json
@@ -0,0 +1,77 @@
+{
+ "uuid": "970b1e03-75fd-6152-ce22-7c6d19bb1760",
+ "last_modified": 0,
+ "create_time": 1675998714528,
+ "version": "4.0.0.0",
+ "name": "CUSTOMER",
+ "columns": [
+ {
+ "id": "1",
+ "name": "C_CUSTKEY",
+ "datatype": "bigint",
+ "case_sensitive_name": "c_custkey"
+ },
+ {
+ "id": "2",
+ "name": "C_NAME",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_name"
+ },
+ {
+ "id": "3",
+ "name": "C_ADDRESS",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_address"
+ },
+ {
+ "id": "4",
+ "name": "C_CITY",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_city"
+ },
+ {
+ "id": "5",
+ "name": "C_NATION",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_nation"
+ },
+ {
+ "id": "6",
+ "name": "C_REGION",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_region"
+ },
+ {
+ "id": "7",
+ "name": "C_PHONE",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_phone"
+ },
+ {
+ "id": "8",
+ "name": "C_MKTSEGMENT",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "c_mktsegment"
+ }
+ ],
+ "source_type": 9,
+ "table_type": "EXTERNAL",
+ "top": false,
+ "increment_loading": false,
+ "last_snapshot_path": null,
+ "last_snapshot_size": 0,
+ "snapshot_last_modified": 0,
+ "query_hit_count": 0,
+ "partition_column": null,
+ "snapshot_partitions": {},
+ "snapshot_partitions_info": {},
+ "snapshot_total_rows": 0,
+ "snapshot_partition_col": null,
+ "selected_snapshot_partition_col": null,
+ "temp_snapshot_path": null,
+ "snapshot_has_broken": false,
+ "database": "SSB",
+ "transactional": false,
+ "rangePartition": false,
+ "partition_desc": null
+}
diff --git a/src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.LINEORDER.json b/src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.LINEORDER.json
new file mode 100644
index 0000000000..08bda05859
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/index_build_test/table/SSB.LINEORDER.json
@@ -0,0 +1,131 @@
+{
+ "uuid": "33529927-8bc2-d0fb-33a3-562db986be6b",
+ "last_modified": 0,
+ "create_time": 1675998714544,
+ "version": "4.0.0.0",
+ "name": "LINEORDER",
+ "columns": [
+ {
+ "id": "1",
+ "name": "LO_ORDERKEY",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_orderkey"
+ },
+ {
+ "id": "2",
+ "name": "LO_LINENUMBER",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_linenumber"
+ },
+ {
+ "id": "3",
+ "name": "LO_CUSTKEY",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_custkey"
+ },
+ {
+ "id": "4",
+ "name": "LO_PARTKEY",
+ "datatype": "integer",
+ "case_sensitive_name": "lo_partkey"
+ },
+ {
+ "id": "5",
+ "name": "LO_SUPPKEY",
+ "datatype": "integer",
+ "case_sensitive_name": "lo_suppkey"
+ },
+ {
+ "id": "6",
+ "name": "LO_ORDERDATE",
+ "datatype": "date",
+ "case_sensitive_name": "lo_orderdate"
+ },
+ {
+ "id": "7",
+ "name": "LO_ORDERPRIOTITY",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "lo_orderpriotity"
+ },
+ {
+ "id": "8",
+ "name": "LO_SHIPPRIOTITY",
+ "datatype": "integer",
+ "case_sensitive_name": "lo_shippriotity"
+ },
+ {
+ "id": "9",
+ "name": "LO_QUANTITY",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_quantity"
+ },
+ {
+ "id": "10",
+ "name": "LO_EXTENDEDPRICE",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_extendedprice"
+ },
+ {
+ "id": "11",
+ "name": "LO_ORDTOTALPRICE",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_ordtotalprice"
+ },
+ {
+ "id": "12",
+ "name": "LO_DISCOUNT",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_discount"
+ },
+ {
+ "id": "13",
+ "name": "LO_REVENUE",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_revenue"
+ },
+ {
+ "id": "14",
+ "name": "LO_SUPPLYCOST",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_supplycost"
+ },
+ {
+ "id": "15",
+ "name": "LO_TAX",
+ "datatype": "bigint",
+ "case_sensitive_name": "lo_tax"
+ },
+ {
+ "id": "16",
+ "name": "LO_COMMITDATE",
+ "datatype": "date",
+ "case_sensitive_name": "lo_commitdate"
+ },
+ {
+ "id": "17",
+ "name": "LO_SHIPMODE",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "lo_shipmode"
+ }
+ ],
+ "source_type": 9,
+ "table_type": "EXTERNAL",
+ "top": false,
+ "increment_loading": false,
+ "last_snapshot_path": null,
+ "last_snapshot_size": 0,
+ "snapshot_last_modified": 0,
+ "query_hit_count": 0,
+ "partition_column": null,
+ "snapshot_partitions": {},
+ "snapshot_partitions_info": {},
+ "snapshot_total_rows": 0,
+ "snapshot_partition_col": null,
+ "selected_snapshot_partition_col": null,
+ "temp_snapshot_path": null,
+ "snapshot_has_broken": false,
+ "database": "SSB",
+ "transactional": false,
+ "rangePartition": false,
+ "partition_desc": null
+}
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NIndexPlanController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NIndexPlanController.java
index 24124071a5..fe5bdc8cfc 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NIndexPlanController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NIndexPlanController.java
@@ -48,6 +48,8 @@ import org.apache.kylin.rest.response.TableIndexResponse;
import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.IndexPlanService;
import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.params.IndexPlanParams;
+import org.apache.kylin.rest.service.params.PaginationParams;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.web.bind.annotation.DeleteMapping;
@@ -178,6 +180,7 @@ public class NIndexPlanController extends NBasicController {
@GetMapping(value = "/index")
public EnvelopeResponse<FusionRuleDataResult<List<IndexResponse>>> getIndex(
@RequestParam(value = "project") String project, @RequestParam(value = "model") String modelId, //
+ @RequestParam(value = "segment_id", required = false, defaultValue = "") String segmentId,
@RequestParam(value = "sort_by", required = false, defaultValue = "") String order,
@RequestParam(value = "reverse", required = false, defaultValue = "false") Boolean desc,
@RequestParam(value = "sources", required = false, defaultValue = "") List<IndexEntity.Source> sources,
@@ -189,7 +192,9 @@ public class NIndexPlanController extends NBasicController {
@RequestParam(value = "range", required = false, defaultValue = "") List<IndexEntity.Range> range) {
checkProjectName(project);
checkRequiredArg(MODEL_ID, modelId);
- val indexes = fusionIndexService.getIndexes(project, modelId, key, status, order, desc, sources, ids, range);
+ IndexPlanParams indexPlanParams = new IndexPlanParams(project, modelId, segmentId, ids, sources, status, range);
+ PaginationParams paginationParams = new PaginationParams(offset, limit, order, desc);
+ val indexes = fusionIndexService.getIndexes(indexPlanParams, paginationParams, key);
val indexUpdateEnabled = FusionIndexService.checkUpdateIndexEnabled(project, modelId);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
FusionRuleDataResult.get(indexes, offset, limit, indexUpdateEnabled), "");
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/IndexResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/IndexResponse.java
index b437c2f857..4ae77b249a 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/IndexResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/IndexResponse.java
@@ -19,6 +19,7 @@ package org.apache.kylin.rest.response;
import java.util.List;
+import org.apache.kylin.metadata.cube.model.NDataLayout;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexEntity.Source;
@@ -101,6 +102,9 @@ public class IndexResponse {
}
+ @JsonProperty("abnormal_type")
+ private NDataLayout.AbnormalType abnormalType;
+
@Data
@AllArgsConstructor
public static class ColOrderPair {
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataSegmentResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataSegmentResponse.java
index 47eb6a6140..8af94f0868 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataSegmentResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataSegmentResponse.java
@@ -120,14 +120,14 @@ public class NDataSegmentResponse extends NDataSegment {
startTime = Long.parseLong(getSegRange().getStart().toString());
endTime = Long.parseLong(getSegRange().getEnd().toString());
storage = bytesSize;
- indexCount = segment.getLayoutSize();
+ indexCount = segment.getWorkingLayoutSize();
indexCountTotal = segment.getIndexPlan().getAllLayoutsSize(true);
multiPartitionCount = segment.getMultiPartitions().size();
hasBaseAggIndex = segment.getIndexPlan().containBaseAggLayout();
hasBaseTableIndex = segment.getIndexPlan().containBaseTableLayout();
if (segment.getIndexPlan().getBaseTableLayout() != null) {
val indexPlan = segment.getDataflow().getIndexPlan();
- long segmentFileCount = segment.getSegDetails().getLayouts().stream()
+ long segmentFileCount = segment.getSegDetails().getWorkingLayouts().stream()
.filter(layout -> indexPlan.getLayoutEntity(layout.getLayoutId()) != null
&& indexPlan.getLayoutEntity(layout.getLayoutId()).isBaseIndex())
.mapToLong(NDataLayout::getFileCount).sum();
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
index 80043d3fd5..b048cd3815 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
@@ -55,6 +55,8 @@ import org.apache.kylin.rest.response.AggIndexResponse;
import org.apache.kylin.rest.response.BuildIndexResponse;
import org.apache.kylin.rest.response.DiffRuleBasedIndexResponse;
import org.apache.kylin.rest.response.IndexResponse;
+import org.apache.kylin.rest.service.params.IndexPlanParams;
+import org.apache.kylin.rest.service.params.PaginationParams;
import org.apache.kylin.streaming.manager.StreamingJobManager;
import org.apache.kylin.streaming.metadata.StreamingJobMeta;
import org.springframework.beans.factory.annotation.Autowired;
@@ -195,10 +197,24 @@ public class FusionIndexService extends BasicService {
}
public List<IndexResponse> getIndexes(String project, String modelId, String key, List<IndexEntity.Status> status,
- String orderBy, Boolean desc, List<IndexEntity.Source> sources, List<Long> ids,
- List<IndexEntity.Range> range) {
- List<IndexResponse> indexes = indexPlanService.getIndexes(project, modelId, key, status, orderBy, desc,
- sources);
+ String orderBy, Boolean desc, List<IndexEntity.Source> sources, List<Long> ids,
+ List<IndexEntity.Range> range) {
+ return getIndexes(new IndexPlanParams(project, modelId, null, ids, sources, status, range),
+ new PaginationParams(null, null, orderBy, desc),
+ key);
+ }
+
+ public List<IndexResponse> getIndexes(IndexPlanParams indexPlanParams, PaginationParams paginationParams, String key) {
+ String project = indexPlanParams.getProject();
+ String modelId = indexPlanParams.getModelId();
+ List<Long> ids = indexPlanParams.getIds();
+ List<IndexEntity.Source> sources = indexPlanParams.getSources();
+ List<IndexEntity.Status> status = indexPlanParams.getStatus();
+ List<IndexEntity.Range> range = indexPlanParams.getRange();
+ String orderBy = paginationParams.getOrderBy();
+ Boolean desc = paginationParams.getReverse();
+
+ List<IndexResponse> indexes = indexPlanService.getIndexes(indexPlanParams, paginationParams, key);
NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
if (model.isFusionModel()) {
FusionModel fusionModel = getManager(FusionModelManager.class, project).getFusionModel(modelId);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
index 1b9e681ef6..767b098dd5 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
@@ -98,6 +98,8 @@ import org.apache.kylin.rest.response.IndexGraphResponse;
import org.apache.kylin.rest.response.IndexResponse;
import org.apache.kylin.rest.response.IndexStatResponse;
import org.apache.kylin.rest.response.TableIndexResponse;
+import org.apache.kylin.rest.service.params.IndexPlanParams;
+import org.apache.kylin.rest.service.params.PaginationParams;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.SpringContext;
import org.slf4j.Logger;
@@ -651,7 +653,21 @@ public class IndexPlanService extends BasicService implements TableIndexPlanSupp
}
public List<IndexResponse> getIndexes(String project, String modelId, String key, List<IndexEntity.Status> status,
- String orderBy, Boolean desc, List<IndexEntity.Source> sources) {
+ String orderBy, Boolean desc, List<IndexEntity.Source> sources) {
+ return getIndexes(new IndexPlanParams(project, modelId, null, null, sources, status, null),
+ new PaginationParams(null, null, orderBy, desc),
+ key);
+ }
+
+ public List<IndexResponse> getIndexes(IndexPlanParams indexPlanParams, PaginationParams paginationParams, String key) {
+ String project = indexPlanParams.getProject();
+ String modelId = indexPlanParams.getModelId();
+ String segmentId = indexPlanParams.getSegmentId();
+ List<IndexEntity.Source> sources = indexPlanParams.getSources();
+ List<IndexEntity.Status> status = indexPlanParams.getStatus();
+ String orderBy = paginationParams.getOrderBy();
+ Boolean desc = paginationParams.getReverse();
+
aclEvaluate.checkProjectReadPermission(project);
Set<IndexEntity.Status> statusSet = Sets.newHashSet(status);
@@ -662,7 +678,7 @@ public class IndexPlanService extends BasicService implements TableIndexPlanSupp
Set<Long> layoutsByRunningJobs = getLayoutsByRunningJobs(project, modelId);
if (StringUtils.isBlank(key)) {
return sortAndFilterLayouts(layouts.stream()
- .map(layoutEntity -> convertToResponse(layoutEntity, indexPlan.getModel(), layoutsByRunningJobs))
+ .map(layoutEntity -> convertToResponse(layoutEntity, indexPlan.getModel(), layoutsByRunningJobs, segmentId))
.filter(indexResponse -> statusSet.isEmpty() || statusSet.contains(indexResponse.getStatus())),
orderBy, desc, sources);
}
@@ -676,7 +692,7 @@ public class IndexPlanService extends BasicService implements TableIndexPlanSupp
return String.valueOf(index.getId()).equals(key.trim())
|| !Sets.intersection(matchDimensions, colOrderSet).isEmpty()
|| !Sets.intersection(matchMeasures, colOrderSet).isEmpty();
- }).map(layoutEntity -> convertToResponse(layoutEntity, indexPlan.getModel(), layoutsByRunningJobs))
+ }).map(layoutEntity -> convertToResponse(layoutEntity, indexPlan.getModel(), layoutsByRunningJobs, segmentId))
.filter(indexResponse -> statusSet.isEmpty() || statusSet.contains(indexResponse.getStatus())), orderBy,
desc, sources);
}
@@ -735,7 +751,7 @@ public class IndexPlanService extends BasicService implements TableIndexPlanSupp
for (NDataSegment seg : readySegments) {
val lockedIndexCountInSeg = seg.getLayoutsMap().values().stream()
.filter(nDataLayout -> nDataLayout.getLayout().isToBeDeleted()).count();
- if ((seg.getSegDetails().getLayouts().size() - lockedIndexCountInSeg) != allIndexCountWithoutTobeDel) {
+ if ((seg.getSegDetails().getAllLayouts().size() - lockedIndexCountInSeg) != allIndexCountWithoutTobeDel) {
segmentToComplementCount += 1;
}
}
@@ -840,11 +856,11 @@ public class IndexPlanService extends BasicService implements TableIndexPlanSupp
}
private IndexResponse convertToResponse(LayoutEntity layoutEntity, NDataModel model) {
- return convertToResponse(layoutEntity, model, Sets.newHashSet());
+ return convertToResponse(layoutEntity, model, Sets.newHashSet(), null);
}
private IndexResponse convertToResponse(LayoutEntity layoutEntity, NDataModel model,
- Set<Long> layoutIdsOfRunningJobs) {
+ Set<Long> layoutIdsOfRunningJobs, String segmentId) {
// remove all internal measures
val colOrders = Lists.newArrayList(layoutEntity.getColOrder());
@@ -862,20 +878,32 @@ public class IndexPlanService extends BasicService implements TableIndexPlanSupp
val dataflow = dfMgr.getDataflow(layoutEntity.getIndex().getIndexPlan().getUuid());
long dataSize = 0L;
int readyCount = 0;
- for (NDataSegment segment : dataflow.getSegments()) {
- val dataCuboid = segment.getLayout(layoutEntity.getId());
- if (dataCuboid == null) {
+ boolean hasDataInconsistent = false;
+
+ List<NDataSegment> segments = StringUtils.isBlank(segmentId) ? dataflow.getSegments()
+ : dataflow.getSegments().stream().filter(seg -> seg.getId().equals(segmentId)).collect(Collectors.toList());
+ for (NDataSegment segment : segments) {
+ NDataLayout layout = segment.getLayout(layoutEntity.getId(), true);
+ if (layout == null) {
+ continue;
+ }
+ if (layout.getAbnormalType() == NDataLayout.AbnormalType.DATA_INCONSISTENT) {
+ hasDataInconsistent = true;
continue;
}
readyCount++;
- dataSize += dataCuboid.getByteSize();
+ dataSize += layout.getByteSize();
}
IndexEntity.Status status;
if (readyCount <= 0) {
- status = IndexEntity.Status.NO_BUILD;
if (layoutIdsOfRunningJobs.contains(layoutEntity.getId())) {
status = IndexEntity.Status.BUILDING;
+ } else {
+ status = IndexEntity.Status.NO_BUILD;
+ if (hasDataInconsistent) {
+ response.setAbnormalType(NDataLayout.AbnormalType.DATA_INCONSISTENT);
+ }
}
} else {
status = IndexEntity.Status.ONLINE;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index de7d4191f4..9a917fb61c 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -1177,7 +1177,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
boolean allToComplement, Set<Long> allIndexWithoutTobeDel, NDataSegment segment) {
if (allToComplement) {
// find seg that does not have all indexes(don't include tobeDeleted)
- val segLayoutIds = segment.getSegDetails().getLayouts().stream().map(NDataLayout::getLayoutId)
+ val segLayoutIds = segment.getSegDetails().getWorkingLayouts().stream().map(NDataLayout::getLayoutId)
.collect(Collectors.toSet());
return !Sets.difference(allIndexWithoutTobeDel, segLayoutIds).isEmpty();
}
@@ -2386,7 +2386,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
for (String segmentId : segmentIds) {
NDataSegment seg = dataflow.getSegment(segmentId);
NDataSegDetails segDetails = seg.getSegDetails();
- List<NDataLayout> layouts = new LinkedList<>(segDetails.getLayouts());
+ List<NDataLayout> layouts = new LinkedList<>(segDetails.getAllLayouts());
layouts.removeIf(layout -> indexIds.contains(layout.getLayoutId()));
dfManger.updateDataflowDetailsLayouts(seg, layouts);
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IndexPlanParams.java
similarity index 58%
copy from src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
copy to src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IndexPlanParams.java
index a8cf9dc237..ea917984d8 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IndexPlanParams.java
@@ -16,21 +16,24 @@
* limitations under the License.
*/
-package org.apache.kylin.engine.spark.job.stage.build
+package org.apache.kylin.rest.service.params;
-import org.apache.kylin.engine.spark.job.SegmentJob
-import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.metadata.cube.model.NDataSegment
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
-class BuildLayer(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
- extends BuildStage(jobContext, dataSegment, buildParam) {
+import java.util.List;
- override def execute(): Unit = {
- // Build layers.
- buildLayouts()
- // Drain results immediately after building.
- drain()
- }
-
- override def getStageName: String = "BuildLayer"
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class IndexPlanParams {
+ private String project;
+ private String modelId;
+ private String segmentId;
+ private List<Long> ids;
+ private List<IndexEntity.Source> sources;
+ private List<IndexEntity.Status> status;
+ private List<IndexEntity.Range> range;
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/PaginationParams.java
similarity index 58%
copy from src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
copy to src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/PaginationParams.java
index a8cf9dc237..b0b495f3bc 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/PaginationParams.java
@@ -16,21 +16,18 @@
* limitations under the License.
*/
-package org.apache.kylin.engine.spark.job.stage.build
+package org.apache.kylin.rest.service.params;
-import org.apache.kylin.engine.spark.job.SegmentJob
-import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.metadata.cube.model.NDataSegment
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
-class BuildLayer(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
- extends BuildStage(jobContext, dataSegment, buildParam) {
-
- override def execute(): Unit = {
- // Build layers.
- buildLayouts()
- // Drain results immediately after building.
- drain()
- }
-
- override def getStageName: String = "BuildLayer"
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class PaginationParams {
+ private Integer pageOffset;
+ private Integer pageSize;
+ private String orderBy;
+ private Boolean reverse;
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
index 70936d305e..be0b4814f7 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
@@ -235,14 +235,14 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
public void testGetReadableProjects() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getReadableProjects("", false);
- Assert.assertEquals(28, projectInstances.size());
+ Assert.assertEquals(29, projectInstances.size());
}
@Test
public void testGetAdminProjects() throws Exception {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getAdminProjects();
- Assert.assertEquals(28, projectInstances.size());
+ Assert.assertEquals(29, projectInstances.size());
}
@Test
@@ -256,7 +256,7 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
public void testGetReadableProjectsHasNoPermissionProject() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getReadableProjects("", false);
- Assert.assertEquals(28, projectInstances.size());
+ Assert.assertEquals(29, projectInstances.size());
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
index 2c46e619ea..0ce477be53 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
@@ -364,7 +364,7 @@ public class QueryHistoryServiceTest extends NLocalFileMetadataTestCase {
// get all tables
tableMap = queryHistoryService.getQueryHistoryTableMap(null);
- Assert.assertEquals(28, tableMap.size());
+ Assert.assertEquals(29, tableMap.size());
// not existing project
tableMap = queryHistoryService.getQueryHistoryTableMap(Lists.newArrayList("not_existing_project"));
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
index 93fe27494f..c6b2848523 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
@@ -217,7 +217,7 @@ public class AsyncQueryJobTest extends NLocalFileMetadataTestCase {
rawResourceMap.put(zipEntry.getName(), raw);
}
}
- Assert.assertEquals(84, rawResourceMap.size());
+ Assert.assertEquals(85, rawResourceMap.size());
}
private void testKylinConfig(FileSystem workingFileSystem, FileStatus metaFileStatus) throws IOException {
diff --git a/src/spark-project/engine-spark/pom.xml b/src/spark-project/engine-spark/pom.xml
index b22aefa720..6af33ecf15 100644
--- a/src/spark-project/engine-spark/pom.xml
+++ b/src/spark-project/engine-spark/pom.xml
@@ -177,6 +177,11 @@
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index cb395e88ba..a7b72aa242 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -35,7 +35,9 @@ import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -71,6 +73,7 @@ import org.apache.kylin.engine.spark.job.UdfManager;
import org.apache.kylin.engine.spark.scheduler.ClusterMonitor;
import org.apache.kylin.engine.spark.utils.JobMetricsUtils;
import org.apache.kylin.engine.spark.utils.SparkConfHelper;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
@@ -118,6 +121,7 @@ public abstract class SparkApplication implements Application {
protected String project;
protected int layoutSize = -1;
protected BuildJobInfos infos;
+ protected ConcurrentHashMap<String, Boolean> skipFollowingStagesMap = new ConcurrentHashMap<>();
/**
* path for spark app args on HDFS
*/
@@ -215,6 +219,17 @@ public abstract class SparkApplication implements Application {
return clusterManager.getBuildTrackingUrl(sparkSession);
}
+ public void setSkipFollowingStages(String segmentId) {
+ skipFollowingStagesMap.put(segmentId, true);
+ }
+
+ public boolean isSkipFollowingStages(String segmentId) {
+ if (segmentId == null) {
+ return false;
+ }
+ return Optional.ofNullable(skipFollowingStagesMap.get(segmentId)).orElse(false);
+ }
+
private String tryReplaceHostAddress(String url) {
String originHost = null;
try {
@@ -446,7 +461,7 @@ public abstract class SparkApplication implements Application {
protected void waiteForResourceSuccess() throws Exception {
val waiteForResource = WAITE_FOR_RESOURCE.create(this, null, null);
- waiteForResource.onStageFinished(true);
+ waiteForResource.onStageFinished(ExecutableState.SUCCEED);
infos.recordStageId("");
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/builder/PartitionDictionaryBuilderHelper.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/builder/PartitionDictionaryBuilderHelper.java
index cd4fcf341e..2d376b51bc 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/builder/PartitionDictionaryBuilderHelper.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/builder/PartitionDictionaryBuilderHelper.java
@@ -49,7 +49,7 @@ public class PartitionDictionaryBuilderHelper extends DictionaryBuilderHelper {
.filter(partition -> !partition.getStatus().equals(PartitionStatusEnum.READY))
.collect(Collectors.toSet());
if (CollectionUtils.isEmpty(newPartitions)) {
- for (NDataLayout cuboid : seg.getSegDetails().getLayouts()) {
+ for (NDataLayout cuboid : seg.getSegDetails().getWorkingLayouts()) {
buildedLayouts.add(cuboid.getLayout());
}
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java
index 0f1b68cd2d..43e9e521f5 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java
@@ -18,21 +18,24 @@
package org.apache.kylin.engine.spark.job;
-import com.google.common.base.Preconditions;
-import lombok.val;
+import java.util.LinkedHashSet;
+import java.util.Optional;
+import java.util.Set;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.merger.AfterBuildResourceMerger;
import org.apache.kylin.job.execution.DefaultExecutableOnModel;
import org.apache.kylin.job.execution.ExecutableHandler;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.LinkedHashSet;
-import java.util.Optional;
-import java.util.Set;
+import com.google.common.base.Preconditions;
+
+import lombok.val;
public class ExecutableAddCuboidHandler extends ExecutableHandler {
private static final Logger logger = LoggerFactory.getLogger(ExecutableAddCuboidHandler.class);
@@ -59,6 +62,28 @@ public class ExecutableAddCuboidHandler extends ExecutableHandler {
.filter(task -> ((NSparkExecutable) task).needMergeMetadata())
.forEach(task -> ((NSparkExecutable) task).mergerMetadata(merger));
+ tryRemoveToBeDeletedLayouts(project, modelId, jobId, executable);
+ markDFStatus();
+ }
+
+ @Override
+ public void handleDiscardOrSuicidal() {
+ val job = getExecutable();
+ // anyTargetSegmentExists && checkCuttingInJobByModel need restart job
+ if (!(job.checkCuttingInJobByModel() && job.checkAnyTargetSegmentAndPartitionExists())) {
+ return;
+ }
+ }
+
+ private void tryRemoveToBeDeletedLayouts(String project, String modelId, String jobId, DefaultExecutableOnModel executable) {
+ if (!(executable instanceof NSparkCubingJob)) {
+ return;
+ }
+ NSparkCubingJob job = (NSparkCubingJob) executable;
+ if (job.getSparkCubingStep().getStatus() != ExecutableState.SUCCEED) {
+ return;
+ }
+
Optional.ofNullable(executable.getParams()).ifPresent(params -> {
String toBeDeletedLayoutIdsStr = params.get(NBatchConstants.P_TO_BE_DELETED_LAYOUT_IDS);
if (StringUtils.isNotBlank(toBeDeletedLayoutIdsStr)) {
@@ -72,16 +97,5 @@ public class ExecutableAddCuboidHandler extends ExecutableHandler {
copyForWrite -> copyForWrite.removeLayouts(toBeDeletedLayoutIds, true, true));
}
});
- markDFStatus();
}
-
- @Override
- public void handleDiscardOrSuicidal() {
- val job = getExecutable();
- // anyTargetSegmentExists && checkCuttingInJobByModel need restart job
- if (!(job.checkCuttingInJobByModel() && job.checkAnyTargetSegmentAndPartitionExists())) {
- return;
- }
- }
-
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingStep.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingStep.java
index 62ea6ec1e7..8f633509a4 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingStep.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingStep.java
@@ -18,23 +18,30 @@
package org.apache.kylin.engine.spark.job;
-import com.google.common.collect.Sets;
-import lombok.NoArgsConstructor;
-import lombok.val;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.spark.merger.MetadataMerger;
import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.execution.NExecutableManager;
+import org.apache.kylin.job.execution.StageBase;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Arrays;
-import java.util.LinkedHashSet;
-import java.util.Set;
+import com.google.common.collect.Sets;
+
+import lombok.NoArgsConstructor;
+import lombok.val;
@NoArgsConstructor
public class NSparkCubingStep extends NSparkExecutable {
@@ -101,6 +108,29 @@ public class NSparkCubingStep extends NSparkExecutable {
return result;
}
+ @Override
+ protected ExecutableState adjustState(ExecutableState originalState) {
+ if (hasWarningStage()) {
+ return ExecutableState.WARNING;
+ }
+ return super.adjustState(originalState);
+ }
+
+ protected boolean hasWarningStage() {
+ NExecutableManager executableManager = getManager();
+ Map<String, List<StageBase>> stagesMap = getStagesMap();
+ for (Map.Entry<String, List<StageBase>> entry : stagesMap.entrySet()) {
+ String segmentId = entry.getKey();
+ List<StageBase> stages = entry.getValue();
+ boolean hasWarning = stages.stream()
+ .anyMatch(stage -> executableManager.getOutput(stage.getId(), segmentId).getState() == ExecutableState.WARNING);
+ if (hasWarning) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static class Mockup {
public static void main(String[] args) {
logger.info(Mockup.class + ".main() invoked, args: " + Arrays.toString(args));
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java
index f0582567f0..d5efe0722a 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java
@@ -110,7 +110,7 @@ public class AfterBuildResourceMerger extends SparkJobMetadataMerger {
theSeg.setStatus(SegmentStatusEnum.READY);
dfUpdate.setToUpdateSegs(theSeg);
dfUpdate.setToRemoveSegs(toRemoveSegments.toArray(new NDataSegment[toRemoveSegments.size()]));
- dfUpdate.setToAddOrUpdateLayouts(theSeg.getSegDetails().getLayouts().toArray(new NDataLayout[0]));
+ dfUpdate.setToAddOrUpdateLayouts(theSeg.getSegDetails().getWorkingLayouts().toArray(new NDataLayout[0]));
localDataflowManager.updateDataflow(dfUpdate);
updateIndexPlan(flowName, remoteStore);
@@ -147,7 +147,7 @@ public class AfterBuildResourceMerger extends SparkJobMetadataMerger {
}
remoteSeg.setLastBuildTime(remoteSeg.getSegDetails().getLastModified());
for (long layoutId : availableLayoutIds) {
- NDataLayout dataCuboid = remoteSeg.getLayout(layoutId);
+ NDataLayout dataCuboid = remoteSeg.getLayout(layoutId, true);
Preconditions.checkNotNull(dataCuboid);
addCuboids.add(dataCuboid);
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
index 3935675a82..8aa4073ae7 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
@@ -83,7 +83,7 @@ public class AfterMergeOrRefreshResourceMerger extends SparkJobMetadataMerger {
partition.setLastBuildTime(lastBuildTime);
});
mergedSegment.setLastBuildTime(lastBuildTime);
- toUpdateCuboids.addAll(new ArrayList<>(mergedSegment.getSegDetails().getLayouts()));
+ toUpdateCuboids.addAll(new ArrayList<>(mergedSegment.getSegDetails().getWorkingLayouts()));
} else {
mergedSegment = upsertSegmentPartition(localSegment, remoteSegment, partitions);
for (val segId : segmentIds) {
@@ -158,7 +158,7 @@ public class AfterMergeOrRefreshResourceMerger extends SparkJobMetadataMerger {
mergedSegment.setStatus(SegmentStatusEnum.WARNING);
}
}
- toUpdateCuboids.addAll(new ArrayList<>(mergedSegment.getSegDetails().getLayouts()));
+ toUpdateCuboids.addAll(new ArrayList<>(mergedSegment.getSegDetails().getWorkingLayouts()));
update.setToAddOrUpdateLayouts(toUpdateCuboids.toArray(new NDataLayout[0]));
update.setToRemoveSegs(toRemoveSegments.toArray(new NDataSegment[0]));
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java
index c179c7d947..698de6d745 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java
@@ -138,7 +138,7 @@ public class DictionaryBuilderHelper {
List<LayoutEntity> buildedLayouts = Lists.newArrayList();
if (seg.getSegDetails() != null) {
- for (NDataLayout cuboid : seg.getSegDetails().getLayouts()) {
+ for (NDataLayout cuboid : seg.getSegDetails().getWorkingLayouts()) {
buildedLayouts.add(cuboid.getLayout());
}
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/BuildJobInfos.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/BuildJobInfos.scala
index 4b2f70329a..b5b193c527 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/BuildJobInfos.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/BuildJobInfos.scala
@@ -24,10 +24,12 @@ import org.apache.spark.application.RetryInfo
import org.apache.spark.sql.execution.SparkPlan
import java.util
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicInteger
class BuildJobInfos {
// BUILD
- private val seg2cuboidsNumPerLayer: util.Map[String, util.List[Int]] = new util.HashMap[String, util.List[Int]]
+ private val seg2cuboidsNumPerLayer: util.Map[String, AtomicInteger] = new ConcurrentHashMap[String, AtomicInteger]()
private val seg2SpanningTree: java.util.Map[String, NSpanningTree] = new util.HashMap[String, NSpanningTree]
@@ -157,22 +159,19 @@ class BuildJobInfos {
}
def recordCuboidsNumPerLayer(segId: String, num: Int): Unit = {
- if (seg2cuboidsNumPerLayer.containsKey(segId)) {
- seg2cuboidsNumPerLayer.get(segId).add(num)
- } else {
- val nums = new util.LinkedList[Int]()
- nums.add(num)
- seg2cuboidsNumPerLayer.put(segId, nums)
+ var counter = seg2cuboidsNumPerLayer.get(segId)
+ if (counter == null) {
+ seg2cuboidsNumPerLayer.putIfAbsent(segId, new AtomicInteger())
+ counter = seg2cuboidsNumPerLayer.get(segId)
}
+ counter.addAndGet(num)
}
def clearCuboidsNumPerLayer(segId: String): Unit = {
- if (seg2cuboidsNumPerLayer.containsKey(segId)) {
- seg2cuboidsNumPerLayer.get(segId).clear()
- }
+ seg2cuboidsNumPerLayer.remove(segId)
}
- def getSeg2cuboidsNumPerLayer: util.Map[String, util.List[Int]] = {
+ def getSeg2cuboidsNumPerLayer: util.Map[String, AtomicInteger] = {
seg2cuboidsNumPerLayer
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/DFMergeJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/DFMergeJob.java
index f3bc59837b..7109e0a297 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/DFMergeJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/DFMergeJob.java
@@ -73,7 +73,7 @@ public class DFMergeJob extends SparkApplication {
// collect layouts need to merge
Map<Long, DFLayoutMergeAssist> mergeCuboidsAssist = Maps.newConcurrentMap();
for (NDataSegment seg : mergingSegments) {
- for (NDataLayout cuboid : seg.getSegDetails().getLayouts()) {
+ for (NDataLayout cuboid : seg.getSegDetails().getWorkingLayouts()) {
long layoutId = cuboid.getLayoutId();
DFLayoutMergeAssist assist = mergeCuboidsAssist.get(layoutId);
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/PartitionExec.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/PartitionExec.scala
index 887b252c01..f9a1fdf075 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/PartitionExec.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/PartitionExec.scala
@@ -91,6 +91,10 @@ private[job] trait PartitionExec {
}
logInfo(s"Segment $segmentId drained layout partition: " + //
s"${results.asScala.map(lp => s"(${lp.layoutId} ${lp.partitionId})").mkString("[", ",", "]")}")
+
+ val buildJobInfos = KylinBuildEnv.get().buildJobInfos
+ buildJobInfos.recordCuboidsNumPerLayer(segmentId, results.size())
+
class DFUpdate extends UnitOfWork.Callback[Int] {
override def process(): Int = {
// Merge into the newest data segment.
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
index e3c930c883..caf4835827 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
@@ -29,10 +29,14 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.spark.builder.SnapshotBuilder;
+import org.apache.kylin.engine.spark.job.LogJobInfoUtils;
+import org.apache.kylin.engine.spark.job.SegmentJob;
+import org.apache.kylin.engine.spark.job.SparkJobConstants;
import org.apache.kylin.engine.spark.job.exec.BuildExec;
import org.apache.kylin.engine.spark.job.stage.BuildParam;
import org.apache.kylin.engine.spark.job.stage.StageExec;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -88,7 +92,7 @@ public class SegmentBuildJob extends SegmentJob {
checkDateFormatIfExist(project, dataflowId);
}
val waiteForResource = WAITE_FOR_RESOURCE.create(this, null, null);
- waiteForResource.onStageFinished(true);
+ waiteForResource.onStageFinished(ExecutableState.SUCCEED);
infos.recordStageId("");
}
@@ -135,8 +139,6 @@ public class SegmentBuildJob extends SegmentJob {
protected void build() throws IOException {
Stream<NDataSegment> segmentStream = config.isSegmentParallelBuildEnabled() ? //
readOnlySegments.parallelStream() : readOnlySegments.stream();
- AtomicLong finishedSegmentCount = new AtomicLong(0);
- val segmentsCount = readOnlySegments.size();
segmentStream.forEach(seg -> {
try (KylinConfig.SetAndUnsetThreadLocalConfig autoCloseConfig = KylinConfig
.setAndUnsetThreadLocalConfig(config)) {
@@ -156,14 +158,10 @@ public class SegmentBuildJob extends SegmentJob {
GATHER_FLAT_TABLE_STATS.createStage(this, seg, buildParam, exec);
BUILD_LAYER.createStage(this, seg, buildParam, exec);
+ REFRESH_COLUMN_BYTES.createStage(this, seg, buildParam, exec);
buildSegment(seg, exec);
- val refreshColumnBytes = REFRESH_COLUMN_BYTES.createStage(this, seg, buildParam, exec);
- refreshColumnBytes.toWorkWithoutFinally();
- if (finishedSegmentCount.incrementAndGet() < segmentsCount) {
- refreshColumnBytes.onStageFinished(true);
- }
} catch (IOException e) {
Throwables.propagate(e);
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala
index e0f635acad..61675ac4a8 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentExec.scala
@@ -18,25 +18,24 @@
package org.apache.kylin.engine.spark.job
-import java.util
-import java.util.Objects
-import java.util.concurrent.{BlockingQueue, ForkJoinPool, LinkedBlockingQueue, TimeUnit}
-
import com.google.common.collect.{Lists, Queues}
-import org.apache.kylin.engine.spark.job.SegmentExec.{LayoutResult, ResultType, SourceStats}
-import org.apache.kylin.engine.spark.job.stage.merge.MergeStage
-import org.apache.kylin.engine.spark.scheduler.JobRuntime
-import org.apache.kylin.metadata.cube.model._
-import org.apache.kylin.metadata.model.NDataModel
import org.apache.hadoop.fs.{Path, PathFilter}
import org.apache.kylin.common.persistence.transaction.UnitOfWork
import org.apache.kylin.common.{KapConfig, KylinConfig}
-import org.apache.kylin.metadata.model.TblColRef
+import org.apache.kylin.engine.spark.job.SegmentExec.{LayoutResult, ResultType, SourceStats, filterSuccessfulLayoutResult}
+import org.apache.kylin.engine.spark.job.stage.merge.MergeStage
+import org.apache.kylin.engine.spark.scheduler.JobRuntime
+import org.apache.kylin.metadata.cube.model.NDataLayout.AbnormalType
+import org.apache.kylin.metadata.cube.model._
+import org.apache.kylin.metadata.model.{NDataModel, TblColRef}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.datasource.storage.{StorageListener, StorageStoreFactory, WriteTaskStats}
import org.apache.spark.sql.{Column, Dataset, Row, SparkSession}
import org.apache.spark.tracker.BuildContext
+import java.util
+import java.util.Objects
+import java.util.concurrent.{BlockingQueue, ForkJoinPool, LinkedBlockingQueue, TimeUnit}
import scala.collection.JavaConverters._
import scala.collection.parallel.ForkJoinTaskSupport
@@ -202,6 +201,9 @@ trait SegmentExec extends Logging {
logInfo(s"Segment $segmentId drained layouts: " + //
s"${results.asScala.map(_.layoutId).mkString("[", ",", "]")}")
+ val buildJobInfos = KylinBuildEnv.get().buildJobInfos
+ buildJobInfos.recordCuboidsNumPerLayer(segmentId, results.asScala.count(filterSuccessfulLayoutResult))
+
class DFUpdate extends UnitOfWork.Callback[Int] {
override def process(): Int = {
@@ -227,6 +229,7 @@ trait SegmentExec extends Logging {
dataLayout.setPartitionValues(taskStats.partitionValues)
dataLayout.setFileCount(taskStats.numFiles)
dataLayout.setByteSize(taskStats.numBytes)
+ dataLayout.setAbnormalType(lr.abnormalType)
dataLayout
}
logInfo(s"Segment $segmentId update the data layouts $dataLayouts")
@@ -290,7 +293,13 @@ trait SegmentExec extends Logging {
val storagePath = NSparkCubingUtil.getStoragePath(segment, layout.getId)
val taskStats = saveWithStatistics(layout, layoutDS, storagePath, readableDesc, storageListener)
val sourceStats = newSourceStats(layout, taskStats)
- pipe.offer(LayoutResult(layout.getId, taskStats, sourceStats))
+ pipe.offer(LayoutResult(layout.getId, taskStats, sourceStats, null))
+ }
+
+ protected final def newEmptyDataLayout(layout: LayoutEntity, abnormalType: AbnormalType): Unit = {
+ val taskStats = WriteTaskStats(0, 0, 0, 0, 0, 0, new util.ArrayList[String]())
+ val sourceStats = SourceStats(0)
+ pipe.offer(LayoutResult(layout.getId, taskStats, sourceStats, abnormalType))
}
protected def newSourceStats(layout: LayoutEntity, taskStats: WriteTaskStats): SourceStats = {
@@ -417,6 +426,11 @@ object SegmentExec {
case class SourceStats(rows: Long)
- case class LayoutResult(layoutId: java.lang.Long, stats: WriteTaskStats, sourceStats: SourceStats) extends ResultType
+ case class LayoutResult(layoutId: java.lang.Long, stats: WriteTaskStats, sourceStats: SourceStats,
+ abnormalType: NDataLayout.AbnormalType) extends ResultType
+
+ protected def filterSuccessfulLayoutResult(layoutResult: LayoutResult): Boolean = {
+ layoutResult.abnormalType == null
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java
index 28174f8d4f..0fd435bea0 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java
@@ -247,4 +247,8 @@ public abstract class SegmentJob extends SparkApplication {
public BuildContext getBuildContext() {
return buildContext;
}
+
+ public IndexPlan getIndexPlan() {
+ return indexPlan;
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java
index 9bb9e12999..9660cf4102 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.job.exec.MergeExec;
import org.apache.kylin.engine.spark.job.stage.BuildParam;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.spark.tracker.BuildContext;
@@ -72,7 +73,7 @@ public class SegmentMergeJob extends SegmentJob {
mergeColumnBytes.toWorkWithoutFinally();
if (finishedSegmentCount.incrementAndGet() < segmentsCount) {
- mergeColumnBytes.onStageFinished(true);
+ mergeColumnBytes.onStageFinished(ExecutableState.SUCCEED);
}
} catch (IOException e) {
Throwables.propagate(e);
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/StageExec.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/StageExec.scala
index 7f99feff77..8eca14faca 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/StageExec.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/StageExec.scala
@@ -19,6 +19,7 @@
package org.apache.kylin.engine.spark.job.stage
import com.google.common.base.Throwables
+import io.kyligence.kap.guava20.shaded.common.util.concurrent.RateLimiter
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.util.JsonUtil
import org.apache.kylin.job.execution.ExecutableState
@@ -44,98 +45,74 @@ trait StageExec extends Logging {
def execute(): Unit
- def onStageStart(): Unit = {
- val taskId = getId
- val segmentId = getSegmentId
- val project = getJobContext.getProject
- val status = ExecutableState.RUNNING.toString
- val errMsg = null
- val updateInfo: util.HashMap[String, String] = null
-
- updateStageInfo(taskId, segmentId, project, status, errMsg, updateInfo)
- }
-
- def updateStageInfo(taskId: String, segmentId: String, project: String, status: String,
- errMsg: String, updateInfo: util.HashMap[String, String]): Unit = {
- val context = getJobContext
-
- val url = "/kylin/api/jobs/stage/status"
-
- val payload: util.HashMap[String, Object] = new util.HashMap[String, Object](6)
- payload.put("task_id", taskId)
- payload.put("segment_id", segmentId)
- payload.put("project", project)
- payload.put("status", status)
- payload.put("err_msg", errMsg)
- payload.put("update_info", updateInfo)
- val json = JsonUtil.writeValueAsString(payload)
- val params = new util.HashMap[String, String]()
- val config = KylinConfig.getInstanceFromEnv
- params.put(ParamsConstants.TIME_OUT, config.getUpdateJobInfoTimeout.toString)
- params.put(ParamsConstants.JOB_TMP_DIR, config.getJobTmpDir(project, true))
- context.getReport.updateSparkJobInfo(params, url, json)
+ def createRateLimiter(permitsPerSecond: Double = 0.1): RateLimiter = {
+ RateLimiter.create(permitsPerSecond)
}
- def onStageFinished(result: Boolean): Unit = {
- val taskId = getId
- val segmentId = getSegmentId
- val project = getJobContext.getProject
- val status = if (result) ExecutableState.SUCCEED.toString else ExecutableState.ERROR.toString
- val errMsg = null
- val updateInfo: util.HashMap[String, String] = null
-
- updateStageInfo(taskId, segmentId, project, status, errMsg, updateInfo)
+ def onStageStart(): Unit = {
+ if (getJobContext.isSkipFollowingStages(getSegmentId)) {
+ return
+ }
+ updateStageInfo(ExecutableState.RUNNING.toString, null, null)
}
- def onBuildLayoutSuccess(layoutCount: Int): Unit = {
- val taskId = getId
- val segmentId = getSegmentId
- val project = getJobContext.getProject
- val status = null
- val errMsg = null
- val updateInfo: util.HashMap[String, String] = new util.HashMap[String, String]
- updateInfo.put(NBatchConstants.P_INDEX_SUCCESS_COUNT, String.valueOf(layoutCount))
-
- updateStageInfo(taskId, segmentId, project, status, errMsg, updateInfo)
+ def onStageFinished(state: ExecutableState = ExecutableState.SUCCEED): Unit = {
+ updateStageInfo(state.toString, null, null)
}
def onStageSkipped(): Unit = {
- val taskId = getId
- val segmentId = getSegmentId
- val project = getJobContext.getProject
- val status = ExecutableState.SKIP.toString
- val errMsg = null
- val updateInfo: util.HashMap[String, String] = null
-
- updateStageInfo(taskId, segmentId, project, status, errMsg, updateInfo)
+ updateStageInfo(ExecutableState.SKIP.toString, null, null)
}
def toWork(): Unit = {
- onStageStart()
- var result: Boolean = false
- try {
- execute()
- result = true
- } catch {
- case throwable: Throwable =>
- KylinBuildEnv.get().buildJobInfos.recordSegmentId(getSegmentId)
- KylinBuildEnv.get().buildJobInfos.recordStageId(getId)
- Throwables.propagate(throwable)
- } finally onStageFinished(result)
+ toWork0()
}
def toWorkWithoutFinally(): Unit = {
+ toWork0(false)
+ }
+
+ def toWork0(doFinally: Boolean = true): Unit = {
onStageStart()
+ var state: ExecutableState = ExecutableState.SUCCEED
try {
+ if (getJobContext.isSkipFollowingStages(getSegmentId)) {
+ state = ExecutableState.SKIP
+ return
+ }
execute()
} catch {
case throwable: Throwable =>
+ state = ExecutableState.ERROR
KylinBuildEnv.get().buildJobInfos.recordSegmentId(getSegmentId)
KylinBuildEnv.get().buildJobInfos.recordStageId(getId)
Throwables.propagate(throwable)
+ } finally {
+ if (doFinally) {
+ onStageFinished(state)
+ }
}
}
+ def updateStageInfo(status: String, errMsg: String, updateInfo: util.Map[String, String]): Unit = {
+ val context = getJobContext
+
+ val url = "/kylin/api/jobs/stage/status"
+
+ val payload: util.HashMap[String, Object] = new util.HashMap[String, Object](6)
+ payload.put("task_id", getId)
+ payload.put("segment_id", getSegmentId)
+ payload.put("project", context.getProject)
+ payload.put("status", status)
+ payload.put("err_msg", errMsg)
+ payload.put("update_info", updateInfo)
+ val json = JsonUtil.writeValueAsString(payload)
+ val params = new util.HashMap[String, String]()
+ val config = KylinConfig.getInstanceFromEnv
+ params.put(ParamsConstants.TIME_OUT, config.getUpdateJobInfoTimeout.toString)
+ params.put(ParamsConstants.JOB_TMP_DIR, config.getJobTmpDir(context.getProject, true))
+ context.getReport.updateSparkJobInfo(params, url, json)
+ }
def setId(id: String): Unit = {
this.id = id
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
index a8cf9dc237..11e71d081a 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildLayer.scala
@@ -18,14 +18,22 @@
package org.apache.kylin.engine.spark.job.stage.build
-import org.apache.kylin.engine.spark.job.SegmentJob
+import io.kyligence.kap.guava20.shaded.common.util.concurrent.RateLimiter
import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.metadata.cube.model.NDataSegment
+import org.apache.kylin.engine.spark.job.{KylinBuildEnv, SegmentJob}
+import org.apache.kylin.metadata.cube.model.{NBatchConstants, NDataSegment}
+
+import java.util.concurrent.TimeUnit
+import scala.collection.JavaConverters._
class BuildLayer(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends BuildStage(jobContext, dataSegment, buildParam) {
+ private val rateLimiter: RateLimiter = createRateLimiter()
+
override def execute(): Unit = {
+ // Start an independent thread doing drain at a fixed rate
+ scheduleCheckpoint()
// Build layers.
buildLayouts()
// Drain results immediately after building.
@@ -33,4 +41,15 @@ class BuildLayer(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam:
}
override def getStageName: String = "BuildLayer"
+
+ override protected def drain(timeout: Long, unit: TimeUnit): Unit = {
+ super.drain(timeout, unit)
+
+ val buildJobInfos = KylinBuildEnv.get().buildJobInfos
+ val layoutCount = buildJobInfos.getSeg2cuboidsNumPerLayer.get(segmentId)
+ if (rateLimiter.tryAcquire() && layoutCount != null) {
+ updateStageInfo(null, null, mapAsJavaMap(Map(NBatchConstants.P_INDEX_SUCCESS_COUNT ->
+ String.valueOf(layoutCount.get()))))
+ }
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
index 414335d9fa..dba42cca96 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
@@ -148,12 +148,6 @@ abstract class BuildStage(private val jobContext: SegmentJob,
override protected def recordTaskInfo(t: Task): Unit = {
logInfo(s"Segment $segmentId submit task: ${t.getTaskDesc}")
- KylinBuildEnv.get().buildJobInfos.recordCuboidsNumPerLayer(segmentId, 1)
- }
-
- override protected def reportTaskProgress(): Unit = {
- val layoutCount = KylinBuildEnv.get().buildJobInfos.getSeg2cuboidsNumPerLayer.get(segmentId).asScala.sum
- onBuildLayoutSuccess(layoutCount)
}
protected def buildStatistics(): Statistics = {
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GatherFlatTableStats.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GatherFlatTableStats.scala
index 6668593a42..a5faa5374e 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GatherFlatTableStats.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GatherFlatTableStats.scala
@@ -28,8 +28,6 @@ class GatherFlatTableStats(jobContext: SegmentJob, dataSegment: NDataSegment, bu
extends BuildStage(jobContext, dataSegment, buildParam) {
override def execute(): Unit = {
- scheduleCheckpoint()
-
// Build flat table?
if (buildParam.getSpanningTree.fromFlatTable()) {
// Collect statistics for flat table.
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTable.scala
index ba62f0bfd9..823079a59b 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTable.scala
@@ -18,26 +18,153 @@
package org.apache.kylin.engine.spark.job.stage.build
-import org.apache.kylin.engine.spark.job.SegmentJob
+import com.google.common.collect.{Maps, Queues}
import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.metadata.cube.model.NDataSegment
+import org.apache.kylin.engine.spark.job.stage.build.GenerateFlatTable._
+import org.apache.kylin.engine.spark.job.{SanityChecker, SegmentJob}
+import org.apache.kylin.job.execution.ExecutableState
+import org.apache.kylin.metadata.cube.model._
+import org.apache.spark.sql.datasource.storage.StorageStoreUtils
import org.apache.spark.sql.{Dataset, Row}
+import scala.collection.JavaConverters._
+
class GenerateFlatTable(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends FlatTableAndDictBase(jobContext, dataSegment, buildParam) {
- override def execute(): Unit = {
- val flatTable: Dataset[Row] = generateFlatTable()
- buildParam.setFlatTable(flatTable)
- val flatTablePart: Dataset[Row] = generateFlatTablePart()
- buildParam.setFlatTablePart(flatTablePart)
+ private var dataCountCheckGood: Boolean = true
+ override def execute(): Unit = {
+ buildParam.setFlatTable(generateFlatTable())
+ buildParam.setFlatTablePart(generateFlatTablePart())
buildParam.setBuildFlatTable(this)
- if (buildParam.isSkipGenerateFlatTable) {
- onStageSkipped()
+ if (!checkDataCountPass()) {
+ drainEmptyLayoutOnDataCountCheckFailed()
+ return
}
}
override def getStageName: String = "GenerateFlatTable"
+
+ override def onStageFinished(state: ExecutableState): Unit = {
+ if (dataCountCheckGood) {
+ super.onStageFinished(state)
+ } else {
+ updateStageInfo(ExecutableState.WARNING.toString, null,
+ Maps.newHashMap(Map(NBatchConstants.P_WARNING_CODE -> NDataLayout.AbnormalType.DATA_INCONSISTENT.name()).asJava))
+ }
+ }
+
+ private def checkDataCountPass(): Boolean = {
+ if (jobContext.getConfig.isDataCountCheckEnabled) {
+ logInfo(s"segment ${dataSegment.getId} dataCountCheck is enabled")
+ val result = checkDataCount()
+ logInfo(s"segment ${dataSegment.getId} dataCountCheck result: ${result}")
+ return result
+ }
+ logInfo(s"segment ${dataSegment.getId} dataCountCheck is not enabled")
+ true
+ }
+
+ private def drainEmptyLayoutOnDataCountCheckFailed(): Unit = {
+ dataCountCheckGood = false
+ jobContext.setSkipFollowingStages(dataSegment.getId)
+ readOnlyLayouts.asScala.foreach(newEmptyDataLayout(_, NDataLayout.AbnormalType.DATA_INCONSISTENT))
+ drain()
+ }
+
+ /**
+ * Data count check for segment layouts before index building, including 2 checks:
+ * <p>Check1: Equality check for sum value of count among existing layouts
+ * (not the layouts that are under construction)
+ * <p>Check2: Equality check for sum value of count between layouts above and flat table
+ *
+ * <p>Special case for check1 is to allow un-equality count sum between agg layouts and table layouts
+ * when <tt>kylin.build.allow-non-strict-count-check</tt> enabled, and should not be used in most cases.
+ *
+ * <p>Check2 will not execute when all the new layouts can be covered by existing layouts in index building.
+ *
+ * <p>Count measure will be used in calculating sum value of count for agg layouts,
+ * and for table layouts simply counting data row numbers.
+ *
+ * @return <tt>true</tt> if data count check pass, <tt>false</tt> otherwise
+ */
+ private def checkDataCount(): Boolean = {
+ val layouts = dataSegment.getSegDetails.getWorkingLayouts.asScala.map(lay => jobContext.getIndexPlan.getLayoutEntity(lay.getLayoutId))
+ val tasks = layouts.map(layout => new DataCountCheckTask(layout, StorageStoreUtils.toDF(dataSegment, layout, sparkSession)))
+ val resultsQueue = Queues.newLinkedBlockingQueue[DataCountCheckResult]()
+
+ if (layouts.isEmpty) return true
+
+ // Start calculating data count
+ slowStartExec(tasks.iterator, (task: DataCountCheckTask) => {
+ val layout = task.layout
+ val ds = task.ds
+ resultsQueue.offer(new DataCountCheckResult(layout, SanityChecker.getCount(ds, layout)))
+ })
+
+ val reduceFor: (DataCountCheckResult => Boolean) => Long = layoutCountReducer(resultsQueue.asScala)
+ val aggLayoutCount = reduceFor(result => !IndexEntity.isTableIndex(result.layout.getId))
+ val tableLayoutCount = reduceFor(result => IndexEntity.isTableIndex(result.layout.getId))
+
+ // All agg layouts count or table layouts count must be same
+ if (isInvalidCount(aggLayoutCount) || isInvalidCount(tableLayoutCount)) {
+ logWarning(s"segment ${dataSegment.getId} dataCountCheck check1 failed, " +
+ s"count number in agg layouts or table layouts are not same, " +
+ s"agg layouts count: ${aggLayoutCount}, table layouts count: ${tableLayoutCount}")
+ return false
+ }
+ // Count number between agg layout and table layout should be same when non-strict count check is not enabled
+ if (bothLayoutsExist(aggLayoutCount, tableLayoutCount)
+ && (aggLayoutCount != tableLayoutCount && !config.isNonStrictCountCheckAllowed)) {
+ logWarning(s"segment ${dataSegment.getId} dataCountCheck check1 failed, " +
+ s"count number between agg layouts and table layouts are not same, " +
+ s"agg layouts count: ${aggLayoutCount}, table layouts count: ${tableLayoutCount}")
+ return false
+ }
+ // Agg layout count equals table layout count, comparing with flat table count or simply return true
+ val layoutCount = if (isLayoutExists(aggLayoutCount)) aggLayoutCount else tableLayoutCount
+ if (isLayoutExists(layoutCount) && buildParam.getSpanningTree.fromFlatTable) {
+ val flatTableCount = buildParam.getFlatTable.count
+ val check2Result = layoutCount == flatTableCount
+ if (!check2Result) {
+ logWarning(s"segment ${dataSegment.getId} dataCountCheck check2 failed, " +
+ s"layouts count: ${layoutCount}, flat table count: ${flatTableCount}")
+ }
+ check2Result
+ } else {
+ true
+ }
+ }
+
+ private def layoutCountReducer(results: Iterable[DataCountCheckResult])(filter: DataCountCheckResult => Boolean): Long = {
+ results.filter(filter)
+ .map(_.count)
+ .reduceOption((prev, cur) => if (prev == cur) cur else InvalidCountFlag)
+ .getOrElse(LayoutNonExistsFlag)
+ }
+
+ sealed class DataCountCheckTask(val layout: LayoutEntity, val ds: Dataset[Row]) extends Task {
+ override def getTaskDesc: String = s"layout ${layout.getId} data count check"
+ }
+
+ sealed class DataCountCheckResult(val layout: LayoutEntity, val count: Long)
+}
+
+object GenerateFlatTable {
+ val InvalidCountFlag: Long = SanityChecker.SKIP_FLAG
+ val LayoutNonExistsFlag: Long = -2L
+
+ def isInvalidCount(count: Long): Boolean = {
+ count == InvalidCountFlag
+ }
+
+ def isLayoutExists(count: Long): Boolean = {
+ count > LayoutNonExistsFlag
+ }
+
+ def bothLayoutsExist(aggLayoutCount: Long, tableLayoutCount: Long): Boolean = {
+ aggLayoutCount > LayoutNonExistsFlag && tableLayoutCount > LayoutNonExistsFlag
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildLayer.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildLayer.scala
index 995bb588cc..621424403a 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildLayer.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildLayer.scala
@@ -18,12 +18,19 @@
package org.apache.kylin.engine.spark.job.stage.build.partition
-import org.apache.kylin.engine.spark.job.SegmentJob
+import io.kyligence.kap.guava20.shaded.common.util.concurrent.RateLimiter
import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.metadata.cube.model.NDataSegment
+import org.apache.kylin.engine.spark.job.{KylinBuildEnv, SegmentJob}
+import org.apache.kylin.metadata.cube.model.{NBatchConstants, NDataSegment}
+
+import java.util.concurrent.TimeUnit
+import scala.collection.JavaConverters._
class PartitionBuildLayer(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends PartitionBuildStage(jobContext, dataSegment, buildParam) {
+
+ private val rateLimiter: RateLimiter = createRateLimiter()
+
override def execute(): Unit = {
// Build layers.
buildLayouts()
@@ -32,4 +39,16 @@ class PartitionBuildLayer(jobContext: SegmentJob, dataSegment: NDataSegment, bui
}
override def getStageName: String = "PartitionBuildLayer"
+
+ override protected def drain(timeout: Long, unit: TimeUnit): Unit = {
+ super.drain(timeout, unit)
+
+ val buildJobInfos = KylinBuildEnv.get().buildJobInfos
+ val layoutCountTotal = buildJobInfos.getSeg2cuboidsNumPerLayer.get(segmentId)
+ if (rateLimiter.tryAcquire() && layoutCountTotal != null) {
+ val layoutCount = layoutCountTotal.get() / partitions.size()
+ updateStageInfo(null, null, mapAsJavaMap(Map(NBatchConstants.P_INDEX_SUCCESS_COUNT ->
+ String.valueOf(layoutCount))))
+ }
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildStage.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildStage.scala
index 331151ecae..6d2bb42355 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildStage.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/partition/PartitionBuildStage.scala
@@ -67,12 +67,6 @@ abstract class PartitionBuildStage(jobContext: SegmentJob, dataSegment: NDataSeg
override protected def columnIdFunc(colRef: TblColRef): String = flatTableDesc.getColumnIdAsString(colRef)
-
- override protected def reportTaskProgress(): Unit = {
- val layoutCount = KylinBuildEnv.get().buildJobInfos.getSeg2cuboidsNumPerLayer.get(segmentId).asScala.sum
- onBuildLayoutSuccess(layoutCount / partitions.size())
- }
-
override protected def buildLayouts(): Unit = {
val taskIter = new BuildTaskIterator[PartitionBuildTask] {
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/MergeStage.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/MergeStage.scala
index 0ac0a14701..54aebff88c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/MergeStage.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/MergeStage.scala
@@ -78,7 +78,7 @@ abstract class MergeStage(private val jobContext: SegmentJob,
// Cleanup previous potentially left temp layout data.
cleanupLayoutTempData(dataSegment, jobContext.getReadOnlyLayouts.asScala.toSeq)
- val tasks = unmerged.flatMap(segment => segment.getSegDetails.getLayouts.asScala) //
+ val tasks = unmerged.flatMap(segment => segment.getSegDetails.getWorkingLayouts.asScala) //
.groupBy(_.getLayoutId).values.map(LayoutMergeTask)
slowStartExec(tasks.iterator, mergeLayout)
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/partition/PartitionMergeStage.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/partition/PartitionMergeStage.scala
index b31f25049c..b932331a8a 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/partition/PartitionMergeStage.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/merge/partition/PartitionMergeStage.scala
@@ -50,7 +50,7 @@ abstract class PartitionMergeStage(private val jobContext: SegmentJob,
override protected def mergeIndices(): Unit = {
val tasks = unmerged.flatMap(segment =>
- segment.getSegDetails.getLayouts.asScala.flatMap(layout =>
+ segment.getSegDetails.getWorkingLayouts.asScala.flatMap(layout =>
layout.getMultiPartition.asScala.map(partition => (layout, partition))
)).groupBy(tp => (tp._1.getLayoutId, tp._2.getPartitionId)).values.map(PartitionMergeTask)
slowStartExec(tasks.iterator, mergePartition)
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTableWithSparkSessionTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTableWithSparkSessionTest.java
new file mode 100644
index 0000000000..8099dc7d4d
--- /dev/null
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/stage/build/GenerateFlatTableWithSparkSessionTest.java
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.spark.job.stage.build;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KapConfig;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.engine.spark.IndexDataConstructor;
+import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
+import org.apache.kylin.engine.spark.job.NSparkCubingJob;
+import org.apache.kylin.engine.spark.job.NSparkCubingUtil;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.execution.NExecutableManager;
+import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
+import org.apache.kylin.metadata.cube.model.LayoutEntity;
+import org.apache.kylin.metadata.cube.model.NDataLayout;
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.kylin.metadata.cube.model.NDataflow;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.datasource.storage.StorageStore;
+import org.apache.spark.sql.datasource.storage.StorageStoreFactory;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import io.kyligence.kap.guava20.shaded.common.collect.Sets;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+@SuppressWarnings("unchecked")
+public class GenerateFlatTableWithSparkSessionTest extends NLocalWithSparkSessionTest {
+
+ @Getter
+ @AllArgsConstructor
+ static class ColumnStruct {
+ private Long rowId;
+ private String rowName;
+ private DataType type;
+ }
+
+ private static final LinkedHashMap<Long, ColumnStruct> DIM_SCHEMAS = new LinkedHashMap<>();
+ private static final LinkedHashMap<Long, ColumnStruct> MEASURE_SCHEMAS = new LinkedHashMap<>();
+ static {
+ DIM_SCHEMAS.put(0L, new ColumnStruct(0L, "LO_ORDERKEY", DataTypes.LongType));
+ DIM_SCHEMAS.put(13L, new ColumnStruct(13L, "LO_QUANTITY", DataTypes.LongType));
+ DIM_SCHEMAS.put(16L, new ColumnStruct(16L, "LO_CUSTKEY", DataTypes.LongType));
+ DIM_SCHEMAS.put(22L, new ColumnStruct(22L, "C_NAME", DataTypes.StringType));
+ DIM_SCHEMAS.put(24L, new ColumnStruct(24L, "C_CUSTKEY", DataTypes.LongType));
+
+ MEASURE_SCHEMAS.put(100000L, new ColumnStruct(100000L, "COUNT_ALL", DataTypes.LongType));
+ MEASURE_SCHEMAS.put(100001L, new ColumnStruct(100001L, "SUM_LINEORDER_LO_QUANTITY", DataTypes.LongType));
+ }
+
+ private static final String MODEL_ID = "3ec47efc-573a-9304-4405-8e05ae184322";
+ private static final String SEGMENT_ID = "b2f206e1-7a15-c94a-20f5-f608d550ead6";
+ private static final long LAYOUT_ID_TO_BUILD = 20001L;
+
+ private KylinConfig config;
+ private NIndexPlanManager indexPlanManager;
+ private NDataflowManager dataflowManager;
+
+ @Override
+ public String getProject() {
+ return "index_build_test";
+ }
+
+ @BeforeAll
+ public static void beforeClass() {
+ NLocalWithSparkSessionTest.beforeClass();
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ NLocalWithSparkSessionTest.afterClass();
+ }
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ super.setUp();
+
+ overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
+ overwriteSystemProp("kylin.engine.persist-flattable-threshold", "0");
+ overwriteSystemProp("kylin.engine.persist-flatview", "true");
+
+ NDefaultScheduler.destroyInstance();
+ NDefaultScheduler scheduler = NDefaultScheduler.getInstance(getProject());
+ scheduler.init(new JobEngineConfig(getTestConfig()));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+
+ config = getTestConfig();
+ indexPlanManager = NIndexPlanManager.getInstance(config, getProject());
+ dataflowManager = NDataflowManager.getInstance(config, getProject());
+ }
+
+ @AfterEach
+ public void tearDown() throws Exception {
+ super.tearDown();
+ NDefaultScheduler.destroyInstance();
+ }
+
+ private Object[] initDataCountCheckTest() {
+ // Enable data count check
+ overwriteSystemProp("kylin.build.data-count-check-enabled", "true");
+ // Index Plan
+ IndexPlan indexPlan = indexPlanManager.getIndexPlan(MODEL_ID);
+ // Segment
+ NDataflow dataflow = dataflowManager.getDataflow(MODEL_ID);
+ NDataSegment segment = dataflow.getSegment(SEGMENT_ID);
+ // Layouts to build
+ Set<Long> layoutIDs = Sets.newHashSet(LAYOUT_ID_TO_BUILD);
+ Set<LayoutEntity> readOnlyLayouts = Collections.unmodifiableSet(NSparkCubingUtil.toLayouts(indexPlan, layoutIDs));
+
+ return new Object[] {indexPlan, dataflow, segment, readOnlyLayouts};
+ }
+
+ @Test
+ void testCheckDataCount_Good() throws Exception {
+ Object[] objs = initDataCountCheckTest();
+ IndexPlan indexPlan = (IndexPlan) objs[0];
+ NDataflow dataflow = (NDataflow) objs[1];
+ NDataSegment segment = (NDataSegment) objs[2];
+ Set<LayoutEntity> readOnlyLayouts = (Set<LayoutEntity>) objs[3];
+
+ // Prepare flat table parquet
+ prepareFlatTableParquet(dataflow, segment, false);
+ // Prepare already existing layouts' parquet
+ prepareLayoutsParquet(indexPlan, segment, false, false, false);
+ // Execute job
+ ExecutableState state = executeJob(segment, readOnlyLayouts);
+
+ // Verify
+ assertEquals(ExecutableState.SUCCEED, state);
+ NDataflow dataflowForVerify = dataflowManager.getDataflow(MODEL_ID);
+ NDataSegment segmentForVerify = dataflowForVerify.getSegment(SEGMENT_ID);
+ NDataLayout layoutForVerify = segmentForVerify.getLayout(LAYOUT_ID_TO_BUILD, true);
+ assertNotNull(layoutForVerify);
+ assertNull(layoutForVerify.getAbnormalType());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "1,10001",
+ "20000000001,20000010001",
+ "1,10001,20000000001,20000010001"
+ })
+ void testCheckDataCount_Good_WithNonExistingLayouts(String nonExistingLayouts) throws Exception {
+ // Enable data count check
+ overwriteSystemProp("kylin.build.data-count-check-enabled", "true");
+
+ Set<Long> nonExistingLayoutsSet = Arrays.stream(nonExistingLayouts.split(",")).map(Long::parseLong)
+ .collect(Collectors.toSet());
+
+ IndexPlan indexPlan = indexPlanManager.getIndexPlan(MODEL_ID);
+ // Layouts to build
+ Set<Long> layoutIDs = Sets.newHashSet(LAYOUT_ID_TO_BUILD);
+ Set<LayoutEntity> readOnlyLayouts = Collections.unmodifiableSet(NSparkCubingUtil.toLayouts(indexPlan, layoutIDs));
+ // Remove all layouts for test
+ indexPlanManager.updateIndexPlan(MODEL_ID, copyForWrite -> {
+ copyForWrite.removeLayouts(nonExistingLayoutsSet, true, true);
+ });
+ dataflowManager.reloadAll();
+ NDataflow dataflow = dataflowManager.getDataflow(MODEL_ID);
+ NDataSegment segment = dataflow.getSegment(SEGMENT_ID);
+
+ // Prepare flat table parquet
+ prepareFlatTableParquet(dataflow, segment, false);
+ // Prepare already existing layouts' parquet
+ prepareLayoutsParquet(indexPlan, segment, false, false, false,
+ nonExistingLayoutsSet);
+ // Execute job
+ ExecutableState state = executeJob(segment, readOnlyLayouts);
+
+ // Verify
+ assertEquals(ExecutableState.SUCCEED, state);
+ NDataflow dataflowForVerify = dataflowManager.getDataflow(MODEL_ID);
+ NDataSegment segmentForVerify = dataflowForVerify.getSegment(SEGMENT_ID);
+ NDataLayout layoutForVerify = segmentForVerify.getLayout(LAYOUT_ID_TO_BUILD, true);
+ assertNotNull(layoutForVerify);
+ assertNull(layoutForVerify.getAbnormalType());
+ }
+
+ @Test
+ void testCheckDataCount_Good_WithNonStrictCountCheckEnabled() throws Exception {
+ Object[] objs = initDataCountCheckTest();
+ IndexPlan indexPlan = (IndexPlan) objs[0];
+ NDataflow dataflow = (NDataflow) objs[1];
+ NDataSegment segment = (NDataSegment) objs[2];
+ Set<LayoutEntity> readOnlyLayouts = (Set<LayoutEntity>) objs[3];
+
+ overwriteSystemProp("kylin.build.allow-non-strict-count-check", "true");
+
+ // Prepare flat table parquet
+ prepareFlatTableParquet(dataflow, segment, false);
+ // Prepare already existing layouts' parquet
+ prepareLayoutsParquet(indexPlan, segment, false, false, true);
+ // Execute job
+ ExecutableState state = executeJob(segment, readOnlyLayouts);
+
+ // Verify
+ assertEquals(ExecutableState.SUCCEED, state);
+ NDataflow dataflowForVerify = dataflowManager.getDataflow(MODEL_ID);
+ NDataSegment segmentForVerify = dataflowForVerify.getSegment(SEGMENT_ID);
+ NDataLayout layoutForVerify = segmentForVerify.getLayout(LAYOUT_ID_TO_BUILD, true);
+ assertNotNull(layoutForVerify);
+ assertNull(layoutForVerify.getAbnormalType());
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "true, false, false",
+ "false, true, false",
+ "false, false, true"
+ })
+ void testCheckDataCount_Failed(boolean loseOneAggLayoutRecord,
+ boolean loseOneTableLayoutRecord,
+ boolean loseTowTableLayoutRecords) throws Exception {
+ Object[] objs = initDataCountCheckTest();
+ IndexPlan indexPlan = (IndexPlan) objs[0];
+ NDataflow dataflow = (NDataflow) objs[1];
+ NDataSegment segment = (NDataSegment) objs[2];
+ Set<LayoutEntity> readOnlyLayouts = (Set<LayoutEntity>) objs[3];
+
+ // Prepare flat table parquet
+ prepareFlatTableParquet(dataflow, segment, false);
+ // Prepare already existing layouts' parquet
+ prepareLayoutsParquet(indexPlan, segment, loseOneAggLayoutRecord, loseOneTableLayoutRecord, loseTowTableLayoutRecords);
+ // Execute job
+ ExecutableState state = executeJob(segment, readOnlyLayouts);
+
+ // Verify
+ assertEquals(ExecutableState.SUCCEED, state);
+ NDataflow dataflowForVerify = dataflowManager.getDataflow(MODEL_ID);
+ NDataSegment segmentForVerify = dataflowForVerify.getSegment(SEGMENT_ID);
+ NDataLayout layoutForVerify = segmentForVerify.getLayout(LAYOUT_ID_TO_BUILD, true);
+ assertNotNull(layoutForVerify);
+ assertEquals(NDataLayout.AbnormalType.DATA_INCONSISTENT, layoutForVerify.getAbnormalType());
+ }
+
+ private void prepareFlatTableParquet(NDataflow dataflow, NDataSegment segment, boolean loseOneRecordForTest) {
+ List<StructField> fields = new ArrayList<>();
+ for (ColumnStruct cs : DIM_SCHEMAS.values()) {
+ StructField field = DataTypes.createStructField(String.valueOf(cs.getRowId()), cs.getType(), true);
+ fields.add(field);
+ }
+ StructType schema = DataTypes.createStructType(fields);
+
+ List<Row> rows = new ArrayList<>();
+ rows.add(RowFactory.create(1L, 5L, 1L, "Customer#000000001", 1L));
+ rows.add(RowFactory.create(2L, 15L, 1L, "Customer#000000001", 1L));
+ rows.add(RowFactory.create(3L, 5L, 2L, "Customer#000000002", 2L));
+ rows.add(RowFactory.create(4L, 15L, 3L, "Customer#000000003", 3L));
+ rows.add(RowFactory.create(5L, 5L, 5L, "Customer#000000005", 5L));
+ if (!loseOneRecordForTest) {
+ rows.add(RowFactory.create(6L, 15L, 5L, "Customer#000000005", 5L));
+ }
+ Dataset<Row> flatTableDS = ss.createDataFrame(rows, schema);
+
+ ss.sessionState().conf().setLocalProperty("spark.sql.sources.repartitionWritingDataSource", "true");
+ flatTableDS.write().mode(SaveMode.Overwrite).parquet(config.getFlatTableDir(getProject(), dataflow.getId(), segment.getId()).toString());
+ }
+
+ private void prepareLayoutsParquet(IndexPlan indexPlan, NDataSegment segment,
+ boolean loseOneAggLayoutRecordForTest,
+ boolean loseOneTableLayoutRecordForTest,
+ boolean loseTwoTableLayoutRecordsForTest) {
+ prepareLayoutsParquet(indexPlan, segment, loseOneAggLayoutRecordForTest, loseOneTableLayoutRecordForTest,
+ loseTwoTableLayoutRecordsForTest, null);
+ }
+
+ private void prepareLayoutsParquet(IndexPlan indexPlan, NDataSegment segment,
+ boolean loseOneAggLayoutRecordForTest,
+ boolean loseOneTableLayoutRecordForTest,
+ boolean loseTwoTableLayoutRecordsForTest,
+ Set<Long> nonExistingLayouts) {
+ StorageStore store = StorageStoreFactory.create(1);
+
+ // Prepare layout 1
+ {
+ List<StructField> fields = new ArrayList<>();
+ for (ColumnStruct cs : toColumnStructs("0 13 16 24 22 100000")) {
+ StructField field = DataTypes.createStructField(String.valueOf(cs.getRowId()), cs.getType(), true);
+ fields.add(field);
+ }
+ StructType schema = DataTypes.createStructType(fields);
+
+ List<Row> rows = new ArrayList<>();
+ rows.add(RowFactory.create(1L, 5L, 1L, 1L, "Customer#000000001", 1L));
+ rows.add(RowFactory.create(2L, 15L, 1L, 1L, "Customer#000000001", 1L));
+ rows.add(RowFactory.create(3L, 5L, 2L, 2L, "Customer#000000002", 1L));
+ rows.add(RowFactory.create(4L, 15L, 3L, 3L, "Customer#000000003", 1L));
+ rows.add(RowFactory.create(5L, 5L, 5L, 5L, "Customer#000000005", 1L));
+ if (!loseOneAggLayoutRecordForTest) {
+ rows.add(RowFactory.create(6L, 15L, 5L, 5L, "Customer#000000005", 1L));
+ }
+
+ Dataset<Row> layoutDS = ss.createDataFrame(rows, schema);
+
+ final long layoutId = 1L;
+ LayoutEntity layoutEntity = indexPlan.getLayoutEntity(layoutId);
+ if (CollectionUtils.isEmpty(nonExistingLayouts) || !nonExistingLayouts.contains(layoutId)) {
+ store.save(layoutEntity, new Path(NSparkCubingUtil.getStoragePath(segment, layoutEntity.getId())), KapConfig.wrap(config), layoutDS);
+ }
+ }
+ // Prepare layout 10001
+ {
+ List<StructField> fields = new ArrayList<>();
+ for (ColumnStruct cs : toColumnStructs("24 22 100000 100001")) {
+ StructField field = DataTypes.createStructField(String.valueOf(cs.getRowId()), cs.getType(), true);
+ fields.add(field);
+ }
+ StructType schema = DataTypes.createStructType(fields);
+
+ List<Row> rows = new ArrayList<>();
+ rows.add(RowFactory.create(1L, "Customer#000000001", 2L, 20L));
+ rows.add(RowFactory.create(2L, "Customer#000000002", 1L, 5L));
+ rows.add(RowFactory.create(3L, "Customer#000000003", 1L, 15L));
+ rows.add(RowFactory.create(5L, "Customer#000000005", 2L, 20L));
+ Dataset<Row> layoutDS = ss.createDataFrame(rows, schema);
+
+ final long layoutId = 10001L;
+ LayoutEntity layoutEntity = indexPlan.getLayoutEntity(layoutId);
+ if (CollectionUtils.isEmpty(nonExistingLayouts) || !nonExistingLayouts.contains(layoutId)) {
+ store.save(layoutEntity, new Path(NSparkCubingUtil.getStoragePath(segment, layoutEntity.getId())), KapConfig.wrap(config), layoutDS);
+ }
+ }
+ // Prepare layout 20000000001
+ {
+ List<StructField> fields = new ArrayList<>();
+ for (ColumnStruct cs : toColumnStructs("0 13 16 24 22")) {
+ StructField field = DataTypes.createStructField(String.valueOf(cs.getRowId()), cs.getType(), true);
+ fields.add(field);
+ }
+ StructType schema = DataTypes.createStructType(fields);
+
+ List<Row> rows = new ArrayList<>();
+ rows.add(RowFactory.create(1L, 5L, 1L, 1L, "Customer#000000001"));
+ rows.add(RowFactory.create(2L, 15L, 1L, 1L, "Customer#000000001"));
+ rows.add(RowFactory.create(3L, 5L, 2L, 2L, "Customer#000000002"));
+ rows.add(RowFactory.create(4L, 15L, 3L, 3L, "Customer#000000003"));
+ rows.add(RowFactory.create(5L, 5L, 5L, 5L, "Customer#000000005"));
+ if (!loseOneTableLayoutRecordForTest && !loseTwoTableLayoutRecordsForTest) {
+ rows.add(RowFactory.create(6L, 15L, 5L, 5L, "Customer#000000005"));
+ }
+ Dataset<Row> layoutDS = ss.createDataFrame(rows, schema);
+
+ final long layoutId = 20_000_000_001L;
+ LayoutEntity layoutEntity = indexPlan.getLayoutEntity(layoutId);
+ if (CollectionUtils.isEmpty(nonExistingLayouts) || !nonExistingLayouts.contains(layoutId)) {
+ store.save(layoutEntity, new Path(NSparkCubingUtil.getStoragePath(segment, layoutEntity.getId())), KapConfig.wrap(config), layoutDS);
+ }
+ }
+ // Prepare layout 20000010001
+ {
+ List<StructField> fields = new ArrayList<>();
+ for (ColumnStruct cs : toColumnStructs("0 13")) {
+ StructField field = DataTypes.createStructField(String.valueOf(cs.getRowId()), cs.getType(), true);
+ fields.add(field);
+ }
+ StructType schema = DataTypes.createStructType(fields);
+
+ List<Row> rows = new ArrayList<>();
+ rows.add(RowFactory.create(1L, 5L));
+ rows.add(RowFactory.create(2L, 15L));
+ rows.add(RowFactory.create(3L, 5L));
+ rows.add(RowFactory.create(4L, 15L));
+ rows.add(RowFactory.create(5L, 5L));
+ if (!loseTwoTableLayoutRecordsForTest) {
+ rows.add(RowFactory.create(6L, 15L));
+ }
+ Dataset<Row> layoutDS = ss.createDataFrame(rows, schema);
+
+ final long layoutId = 20_000_010_001L;
+ LayoutEntity layoutEntity = indexPlan.getLayoutEntity(layoutId);
+ if (CollectionUtils.isEmpty(nonExistingLayouts) || !nonExistingLayouts.contains(layoutId)) {
+ store.save(layoutEntity, new Path(NSparkCubingUtil.getStoragePath(segment, layoutEntity.getId())), KapConfig.wrap(config), layoutDS);
+ }
+ }
+ }
+
+ private List<ColumnStruct> toColumnStructs(String columnIds) {
+ return Arrays.stream(columnIds.split(" ")).map(id -> {
+ ColumnStruct cs = DIM_SCHEMAS.get(Long.valueOf(id));
+ if (cs == null) {
+ cs = MEASURE_SCHEMAS.get(Long.valueOf(id));
+ }
+ return cs;
+ }).collect(Collectors.toList());
+ }
+
+ private ExecutableState executeJob(NDataSegment segment, Set<LayoutEntity> readOnlyLayouts) throws InterruptedException {
+ NExecutableManager executableManager = NExecutableManager.getInstance(config, getProject());
+
+ NSparkCubingJob job = NSparkCubingJob.create(Sets.newHashSet(segment), readOnlyLayouts, "ADMIN", null);
+ executableManager.addJob(job);
+ return IndexDataConstructor.wait(job);
+ }
+}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/DataflowCleanerCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/garbage/DataflowCleanerCLI.java
index 22f2e5f93a..a6390150a1 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/DataflowCleanerCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/garbage/DataflowCleanerCLI.java
@@ -78,7 +78,7 @@ public class DataflowCleanerCLI {
val layoutIds = getLayouts(dataflow);
val toBeRemoved = Sets.<Long> newHashSet();
for (NDataSegment segment : dataflow.getSegments()) {
- toBeRemoved.addAll(segment.getSegDetails().getLayouts().stream().map(NDataLayout::getLayoutId)
+ toBeRemoved.addAll(segment.getSegDetails().getAllLayouts().stream().map(NDataLayout::getLayoutId)
.filter(id -> !layoutIds.contains(id)).collect(Collectors.toSet()));
}
dataflowManager.removeLayouts(dataflow, Lists.newArrayList(toBeRemoved));
[kylin] 21/38: KYLIN-5533 add model list lite api
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2e29f00053bdb99673bf9e103412fc1020099cd7
Author: lixiang <44...@qq.com>
AuthorDate: Sat Feb 25 00:03:27 2023 +0800
KYLIN-5533 add model list lite api
---
.../rest/controller/open/OpenModelController.java | 5 +-
.../kylin/rest/controller/NModelController.java | 11 +-
.../controller/open/OpenModelControllerTest.java | 4 +-
.../rest/controller/NModelControllerTest.java | 6 +-
.../rest/response/NDataModelLiteResponse.java | 129 +++++++++++++++++++++
.../kylin/rest/response/NDataModelResponse.java | 18 +++
.../apache/kylin/rest/service/ModelService.java | 44 ++++---
.../rest/service/params/ModelQueryParams.java | 2 +
.../rest/response/NDataModelResponseTest.java | 106 +++++++++++++++++
.../kylin/rest/service/ModelQueryServiceTest.java | 18 +--
.../kylin/rest/service/ModelServiceQueryTest.java | 99 ++++++++++------
.../kap/secondstorage/SecondStorageLockTest.java | 6 +-
12 files changed, 377 insertions(+), 71 deletions(-)
diff --git a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
index fff6b63367..3b10bfc29a 100644
--- a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
+++ b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
@@ -161,11 +161,12 @@ public class OpenModelController extends NBasicController {
@RequestParam(value = "model_alias_or_owner", required = false) String modelAliasOrOwner,
@RequestParam(value = "last_modify_from", required = false) Long lastModifyFrom,
@RequestParam(value = "last_modify_to", required = false) Long lastModifyTo,
- @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim) {
+ @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim,
+ @RequestParam(value = "lite", required = false, defaultValue = "false") boolean lite) {
String projectName = checkProjectName(project);
return modelController.getModels(modelId, modelAlias, exactMatch, projectName, owner, status, table, offset,
limit, sortBy, reverse, modelAliasOrOwner, Collections.singletonList(ModelAttributeEnum.BATCH),
- lastModifyFrom, lastModifyTo, onlyNormalDim);
+ lastModifyFrom, lastModifyTo, onlyNormalDim, lite);
}
@ApiOperation(value = "getIndexes", tags = { "AI" })
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
index b520231844..e5029e98f9 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
@@ -75,6 +75,7 @@ import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.IndexPlanService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ModelTdsService;
+import org.apache.kylin.rest.service.params.ModelQueryParams;
import org.apache.kylin.tool.bisync.SyncContext;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.util.DataRangeUtils;
@@ -143,13 +144,15 @@ public class NModelController extends NBasicController {
@RequestParam(value = "model_attributes", required = false) List<ModelAttributeEnum> modelAttributes,
@RequestParam(value = "last_modify_from", required = false) Long lastModifyFrom,
@RequestParam(value = "last_modify_to", required = false) Long lastModifyTo,
- @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim) {
+ @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim,
+ @RequestParam(value = "lite", required = false, defaultValue = "false") boolean lite) {
checkProjectName(project);
status = formatStatus(status, ModelStatusToDisplayEnum.class);
- DataResult<List<NDataModel>> filterModels = modelService.getModels(modelId, modelAlias, exactMatch, project,
- owner, status, table, offset, limit, sortBy, reverse, modelAliasOrOwner, modelAttributes,
- lastModifyFrom, lastModifyTo, onlyNormalDim);
+ ModelQueryParams request = new ModelQueryParams(modelId, modelAlias, exactMatch, project, owner, status,
+ table, offset, limit, sortBy, reverse, modelAliasOrOwner, modelAttributes, lastModifyFrom, lastModifyTo,
+ onlyNormalDim, lite);
+ DataResult<List<NDataModel>> filterModels = modelService.getModels(request);
fusionModelService.setModelUpdateEnabled(filterModels);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, filterModels, "");
}
diff --git a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
index ccc7864448..52c97e7629 100644
--- a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
+++ b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
@@ -187,7 +187,7 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetModels() throws Exception {
Mockito.when(nModelController.getModels("model1", "model1", true, "default", "ADMIN", Arrays.asList("NEW"), "",
- 1, 5, "last_modify", false, null, null, null, null, true)).thenReturn(
+ 1, 5, "last_modify", false, null, null, null, null, true, false)).thenReturn(
new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(mockModels(), 0, 10), ""));
mockMvc.perform(MockMvcRequestBuilders.get("/api/models").contentType(MediaType.APPLICATION_JSON)
.param("page_offset", "1").param("project", "default").param("model_id", "model1")
@@ -197,7 +197,7 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(openModelController).getModels("default", "model1", "model1", true, "ADMIN",
- Arrays.asList("NEW"), "", 1, 5, "last_modify", true, null, null, null, true);
+ Arrays.asList("NEW"), "", 1, 5, "last_modify", true, null, null, null, true, false);
}
@Test
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
index 1638aa0bad..ef405752e1 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
@@ -241,7 +241,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
Mockito.verify(nModelController).getModels(null, "model1", true, "default", "ADMIN", Arrays.asList("ONLINE"),
"", 0, 10, "last_modify", true, null, Arrays.asList(ModelAttributeEnum.BATCH,
ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID, ModelAttributeEnum.SECOND_STORAGE),
- null, null, true);
+ null, null, true, false);
}
@Test
@@ -255,7 +255,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(nModelController).getModels(null, "model1", true, "default", "ADMIN", Arrays.asList("ONLINE"),
- "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true);
+ "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true, false);
}
@Test
@@ -269,7 +269,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(nModelController).getModels(null, "", true, "default", "ADMIN", Arrays.asList("ONLINE"),
- "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true);
+ "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true, false);
}
@Test
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java
new file mode 100644
index 0000000000..ebcfd79627
--- /dev/null
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.response;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.springframework.beans.BeanUtils;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@EqualsAndHashCode(callSuper = true)
+public class NDataModelLiteResponse extends NDataModelResponse {
+
+ @JsonProperty("empty_model")
+ private boolean emptyModel;
+
+ @JsonProperty("partition_column_in_dims")
+ private boolean partitionColumnInDims;
+
+ @JsonProperty("batch_id")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private String batchId;
+
+ @JsonProperty("streaming_indexes")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private Long streamingIndexes;
+
+ @JsonProperty("batch_partition_desc")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private PartitionDesc batchPartitionDesc;
+
+ @SuppressWarnings("rawtypes")
+ @JsonProperty("batch_segment_holes")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private List<SegmentRange> batchSegmentHoles;
+
+ @JsonProperty("batch_segments")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private List<NDataSegmentResponse> batchSegments = new ArrayList<>();
+
+ @JsonProperty("simplified_tables")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedTableResponse> getSimpleTables() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("selected_columns")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedNamedColumn> getSelectedColumns() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("simplified_dimensions")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedNamedColumn> getNamedColumns() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("all_named_columns")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<NamedColumn> getAllNamedColumns() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("all_measures")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<Measure> getMeasures() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("simplified_measures")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedMeasure> getSimplifiedMeasures() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("segments")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<NDataSegmentResponse> getSegments() {
+ return Collections.emptyList();
+ }
+
+ public NDataModelLiteResponse(NDataModelResponse response, NDataModel dataModel) {
+ super(dataModel);
+ if (dataModel.isFusionModel()) {
+ FusionModelResponse fusionModelResponse = (FusionModelResponse) response;
+ this.setBatchId(fusionModelResponse.getBatchId());
+ this.setBatchPartitionDesc(fusionModelResponse.getBatchPartitionDesc());
+ this.setStreamingIndexes(fusionModelResponse.getStreamingIndexes());
+ this.setBatchSegmentHoles(fusionModelResponse.getBatchSegmentHoles());
+ }
+ BeanUtils.copyProperties(response, this);
+ }
+}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
index 846bdc7855..cf92dccb13 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
@@ -40,6 +40,7 @@ import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.model.TableRef;
@@ -169,6 +170,23 @@ public class NDataModelResponse extends NDataModel {
.collect(Collectors.toList()));
}
+ @JsonIgnore
+ public boolean isEmptyModel() {
+ List<SimplifiedMeasure> simplifiedMeasures = getSimplifiedMeasures();
+ return getNamedColumns().isEmpty() && simplifiedMeasures.size() == 1
+ && "COUNT_ALL".equals(simplifiedMeasures.get(0).getName());
+ }
+
+ @JsonIgnore
+ public boolean isPartitionColumnInDims() {
+ PartitionDesc partitionDesc = getPartitionDesc();
+ if (partitionDesc == null || partitionDesc.getPartitionDateColumn() == null) {
+ return false;
+ }
+ String partitionColumn = partitionDesc.getPartitionDateColumn();
+ return getNamedColumns().stream().anyMatch(dim -> dim.getAliasDotColumn().equalsIgnoreCase(partitionColumn));
+ }
+
@JsonProperty("simplified_dimensions")
public List<SimplifiedNamedColumn> getNamedColumns() {
if (simplifiedDims != null) {
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 6c5aced159..c3645f4af8 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -199,7 +199,6 @@ import org.apache.kylin.metadata.streaming.KafkaConfig;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.rest.aspect.Transaction;
-import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
import org.apache.kylin.rest.request.ModelConfigRequest;
import org.apache.kylin.rest.request.ModelParatitionDescRequest;
@@ -225,6 +224,7 @@ import org.apache.kylin.rest.response.ModelConfigResponse;
import org.apache.kylin.rest.response.ModelSaveCheckResponse;
import org.apache.kylin.rest.response.MultiPartitionValueResponse;
import org.apache.kylin.rest.response.NCubeDescResponse;
+import org.apache.kylin.rest.response.NDataModelLiteResponse;
import org.apache.kylin.rest.response.NDataModelOldParams;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.NDataSegmentResponse;
@@ -378,7 +378,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
((NDataModelResponse) model).isHasSegments() || CollectionUtils.isNotEmpty(segments));
}
- if (model instanceof FusionModelResponse) {
+ if (model.isFusionModel()) {
FusionModel fusionModel = getManager(FusionModelManager.class, model.getProject())
.getFusionModel(model.getId());
NDataModel batchModel = fusionModel.getBatchModel();
@@ -386,7 +386,9 @@ public class ModelService extends AbstractModelService implements TableModelSupp
List<NDataSegmentResponse> batchSegments = getSegmentsResponse(batchModel.getUuid(), model.getProject(),
"1", String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true);
calculateRecordSizeAndCount(batchSegments, oldParams);
- ((FusionModelResponse) model).setBatchSegments(batchSegments);
+ if (model instanceof FusionModelResponse) {
+ ((FusionModelResponse) model).setBatchSegments(batchSegments);
+ }
}
}
}
@@ -639,18 +641,18 @@ public class ModelService extends AbstractModelService implements TableModelSupp
.filter(NDataModel::isMultiPartitionModel).collect(Collectors.toList());
}
- public DataResult<List<NDataModel>> getModels(String modelId, String modelAlias, boolean exactMatch, String project,
- String owner, List<String> status, String table, Integer offset, Integer limit, String sortBy,
- boolean reverse, String modelAliasOrOwner, List<ModelAttributeEnum> modelAttributes, Long lastModifyFrom,
- Long lastModifyTo, boolean onlyNormalDim) {
+ public DataResult<List<NDataModel>> getModels(ModelQueryParams params) {
List<NDataModel> models = new ArrayList<>();
DataResult<List<NDataModel>> filterModels;
+ val table = params.getTable();
+ val project = params.getProjectName();
+ val modelAttributes = params.getModelAttributes();
+ val modelId = params.getModelId();
+ val offset = params.getOffset();
+ val limit = params.getLimit();
if (StringUtils.isEmpty(table)) {
- val modelQueryParams = new ModelQueryParams(modelId, modelAlias, exactMatch, project, owner, status, offset,
- limit, sortBy, reverse, modelAliasOrOwner, modelAttributes, lastModifyFrom, lastModifyTo,
- onlyNormalDim);
- val tripleList = modelQuerySupporter.getModels(modelQueryParams);
- val pair = getModelsOfCurrentPage(modelQueryParams, tripleList);
+ val tripleList = modelQuerySupporter.getModels(params);
+ val pair = getModelsOfCurrentPage(params, tripleList, params.isLite());
models.addAll(pair.getFirst());
// add second storage infos
ModelUtils.addSecondStorageInfo(project, models);
@@ -659,7 +661,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
filterModels.setValue(updateResponseAcl(filterModels.getValue(), project));
return filterModels;
}
- models.addAll(getRelateModels(project, table, modelAlias));
+ models.addAll(getRelateModels(project, table, params.getModelAlias()));
Set<NDataModel> filteredModels = ModelUtils.getFilteredModels(project, modelAttributes, models);
if (CollectionUtils.isNotEmpty(modelAttributes)) {
@@ -676,7 +678,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public Pair<List<NDataModelResponse>, Integer> getModelsOfCurrentPage(ModelQueryParams queryElem,
- List<ModelTriple> modelTripleList) {
+ List<ModelTriple> modelTripleList, boolean lite) {
val projectName = queryElem.getProjectName();
val offset = queryElem.getOffset();
val limit = queryElem.getLimit();
@@ -691,8 +693,13 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
NDataModel dataModel = t.getDataModel();
try {
- return convertToDataModelResponse(dataModel, projectName, dfManager, status,
- queryElem.isOnlyNormalDim());
+ NDataModelResponse nDataModelResponse = convertToDataModelResponse(dataModel, projectName, dfManager,
+ status, queryElem.isOnlyNormalDim());
+ if (lite) {
+ return new NDataModelLiteResponse(nDataModelResponse, dataModel);
+ } else {
+ return nDataModelResponse;
+ }
} catch (Exception e) {
String errorMsg = String.format(Locale.ROOT,
"convert NDataModelResponse failed, mark to broken. %s, %s", dataModel.getAlias(),
@@ -710,6 +717,11 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return new Pair<>(filterModels, totalSize.get());
}
+ public Pair<List<NDataModelResponse>, Integer> getModelsOfCurrentPage(ModelQueryParams queryElem,
+ List<ModelTriple> modelTripleList) {
+ return getModelsOfCurrentPage(queryElem, modelTripleList, false);
+ }
+
public NDataModelResponse convertToDataModelResponseBroken(NDataModel modelDesc) {
NDataModelResponse response = new NDataModelResponse(modelDesc);
response.setStatus(ModelStatusToDisplayEnum.BROKEN);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java
index e94439e534..b7d9194cc6 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java
@@ -33,6 +33,7 @@ public class ModelQueryParams {
private String projectName;
private String owner;
private List<String> status;
+ private String table;
private Integer offset;
private Integer limit;
private String sortBy;
@@ -42,4 +43,5 @@ public class ModelQueryParams {
private Long lastModifyFrom;
private Long lastModifyTo;
private boolean onlyNormalDim;
+ private boolean lite;
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java
index 231baf8f89..b009be95ef 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java
@@ -21,7 +21,10 @@ package org.apache.kylin.rest.response;
import static com.google.common.collect.Lists.newArrayList;
import static org.apache.kylin.common.util.TestUtils.getTestConfig;
import static org.apache.kylin.metadata.model.FunctionDesc.FUNC_COUNT;
+import static org.apache.kylin.metadata.model.FunctionDesc.FUNC_MIN;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
@@ -30,6 +33,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.util.MetadataTestUtils;
import org.junit.jupiter.api.Assertions;
@@ -170,6 +174,108 @@ class NDataModelResponseTest {
Assertions.assertEquals(1, namedColumns.size());
}
+ @Test
+ void testIsHaveNoDimsAndMeasWithoutCountAll() {
+ List<NDataModel.NamedColumn> allNamedColumns = Lists.newArrayList();
+ NDataModel modelWithoutNamedColumns = new NDataModel();
+ modelWithoutNamedColumns.setUuid("model");
+ modelWithoutNamedColumns.setProject(PROJECT);
+ modelWithoutNamedColumns.setAllNamedColumns(allNamedColumns);
+ modelWithoutNamedColumns.setAllMeasures(Lists.newArrayList(createMeasure()));
+ modelWithoutNamedColumns.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ createModel(modelWithoutNamedColumns);
+ NDataModelResponse withoutColumnResponse = new NDataModelResponse(modelWithoutNamedColumns);
+ Assertions.assertFalse(withoutColumnResponse.isEmptyModel());
+
+ NDataModel model1 = new NDataModel();
+ model1.setProject(PROJECT);
+ model1.setAllNamedColumns(allNamedColumns);
+ model1.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model1.setUuid("model1");
+ model1.setAllMeasures(Arrays.asList(createCountAllMeasure(), createTestMeasure()));
+ createModel(model1);
+ NDataModelResponse model1Response = new NDataModelResponse(model1);
+ Assertions.assertFalse(model1Response.isEmptyModel());
+
+ NDataModel model2 = new NDataModel();
+ model2.setProject(PROJECT);
+ model2.setAllNamedColumns(allNamedColumns);
+ model2.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model2.setUuid("model2");
+ model2.setAllMeasures(Collections.singletonList(createCountAllMeasure()));
+ createModel(model2);
+ NDataModelResponse model2Response = new NDataModelResponse(model2);
+ Assertions.assertTrue(model2Response.isEmptyModel());
+
+ NDataModel model3 = new NDataModel();
+ model3.setProject(PROJECT);
+ model3.setAllNamedColumns(allNamedColumns);
+ model3.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model3.setUuid("model3");
+ model3.setAllMeasures(Collections.singletonList(createMeasure()));
+ createModel(model3);
+ NDataModelResponse model3Response = new NDataModelResponse(model3);
+ Assertions.assertFalse(model3Response.isEmptyModel());
+ }
+
+ @Test
+ void testIsPartitionColumnInDims() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ List<NDataModel.NamedColumn> allNamedColumns = Lists.newArrayList();
+
+ NDataModel model = new NDataModel();
+ model.setUuid("model");
+ model.setProject(PROJECT);
+ model.setAllNamedColumns(allNamedColumns);
+ model.setAllMeasures(Lists.newArrayList(createMeasure()));
+ model.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model.setPartitionDesc(partitionDesc);
+ createModel(model);
+ NDataModelResponse modelResponse = new NDataModelResponse(model);
+ Assertions.assertFalse(modelResponse.isPartitionColumnInDims());
+
+ NDataModel model1 = new NDataModel();
+ model1.setUuid("model1");
+ model1.setProject(PROJECT);
+ model1.setAllNamedColumns(allNamedColumns);
+ model1.setAllMeasures(Lists.newArrayList(createMeasure()));
+ model1.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model1.setPartitionDesc(null);
+ createModel(model1);
+ NDataModelResponse modelResponse1 = new NDataModelResponse(model1);
+ Assertions.assertFalse(modelResponse1.isPartitionColumnInDims());
+
+ NDataModel model2 = new NDataModel();
+ model2.setUuid("model2");
+ model2.setProject(PROJECT);
+ model2.setAllNamedColumns(allNamedColumns);
+ model2.setAllMeasures(Lists.newArrayList(createMeasure()));
+ model2.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ partitionDesc.setPartitionDateColumn("CAL_DT");
+ model2.setPartitionDesc(partitionDesc);
+ createModel(model2);
+ NDataModelResponse modelResponse2 = new NDataModelResponse(model2);
+ Assertions.assertFalse(modelResponse2.isPartitionColumnInDims());
+ }
+
+ private NDataModel.Measure createCountAllMeasure() {
+ NDataModel.Measure countOneMeasure = new NDataModel.Measure();
+ countOneMeasure.setName("COUNT_ALL");
+ countOneMeasure.setFunction(
+ FunctionDesc.newInstance(FUNC_COUNT, newArrayList(ParameterDesc.newInstance("1")), "bigint"));
+ countOneMeasure.setId(200001);
+ return countOneMeasure;
+ }
+
+ private NDataModel.Measure createTestMeasure() {
+ NDataModel.Measure countOneMeasure = new NDataModel.Measure();
+ countOneMeasure.setName("TEST_M");
+ countOneMeasure.setFunction(
+ FunctionDesc.newInstance(FUNC_MIN, newArrayList(ParameterDesc.newInstance("1")), "bigint"));
+ countOneMeasure.setId(1);
+ return countOneMeasure;
+ }
+
private NDataModel.Measure createMeasure() {
NDataModel.Measure countOneMeasure = new NDataModel.Measure();
countOneMeasure.setName("COUNT_ONE");
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java
index efd883b535..a2cf68983e 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java
@@ -22,10 +22,10 @@ import java.util.List;
import java.util.Set;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.service.params.ModelQueryParams;
@@ -77,8 +77,8 @@ public class ModelQueryServiceTest extends NLocalFileMetadataTestCase {
List models = Arrays.asList(mt1);
List<ModelAttributeEnum> modelAttributeSet1 = Lists.newArrayList(ModelAttributeEnum.BATCH);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "", true,
- null, modelAttributeSet1, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet1, null, null, true, false);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isProjectEnable")).toReturn(Boolean.TRUE);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isModelEnable")).toReturn(Boolean.FALSE);
@@ -96,8 +96,8 @@ public class ModelQueryServiceTest extends NLocalFileMetadataTestCase {
List<ModelAttributeEnum> modelAttributeSet1 = Lists.newArrayList(ModelAttributeEnum.STREAMING,
ModelAttributeEnum.HYBRID);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "streaming_test", null, null, 0, 10,
- "", true, null, modelAttributeSet1, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "streaming_test", null, null, "", 0,
+ 10, "", true, null, modelAttributeSet1, null, null, true, false);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isProjectEnable")).toReturn(Boolean.TRUE);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isModelEnable")).toReturn(Boolean.FALSE);
@@ -120,14 +120,14 @@ public class ModelQueryServiceTest extends NLocalFileMetadataTestCase {
List<ModelAttributeEnum> modelAttributeSet1 = Lists.newArrayList(ModelAttributeEnum.BATCH,
ModelAttributeEnum.SECOND_STORAGE);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "", true,
- null, modelAttributeSet1, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet1, null, null, true, false);
List<ModelTriple> filteredModels1 = modelQueryService.filterModels(models, modelQueryParams);
Assert.assertEquals(1, filteredModels1.size());
List<ModelAttributeEnum> modelAttributeSet2 = Lists.newArrayList(ModelAttributeEnum.SECOND_STORAGE);
- ModelQueryParams modelQueryParams2 = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "",
- true, null, modelAttributeSet2, null, null, true);
+ ModelQueryParams modelQueryParams2 = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet2, null, null, true, false);
List<ModelTriple> filteredModels2 = modelQueryService.filterModels(models, modelQueryParams2);
Assert.assertEquals(1, filteredModels2.size());
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
index 351d675346..0955209212 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
@@ -42,6 +42,7 @@ import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.FusionModelResponse;
+import org.apache.kylin.rest.response.NDataModelLiteResponse;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.RelatedModelResponse;
import org.apache.kylin.rest.service.params.ModelQueryParams;
@@ -154,17 +155,27 @@ public class ModelServiceQueryTest extends SourceTestCase {
@Test
public void testQueryModels() {
String project = "streaming_test";
- val modelList = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0, 8,
- "last_modify", true, null,
+ ModelQueryParams request = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0,
+ 8, "last_modify", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, true);
+ null, true, false);
+ val modelList = modelService.getModels(request);
Assert.assertEquals(11, modelList.getTotalSize());
Assert.assertEquals(8, modelList.getValue().size());
- val modelList1 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 10,
- "usage", true, null,
+ ModelQueryParams liteRequest = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(),
+ "", 0, 8, "last_modify", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, true);
+ null, true, true);
+ val modelListWithLite = modelService.getModels(liteRequest);
+ Assert.assertEquals(11, modelListWithLite.getTotalSize());
+ Assert.assertEquals(8, modelListWithLite.getValue().size());
+
+ ModelQueryParams request1 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 10, "usage", true, null,
+ Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
+ null, true, false);
+ val modelList1 = modelService.getModels(request1);
Assert.assertEquals(11, modelList1.getTotalSize());
Assert.assertEquals(1, modelList1.getValue().size());
@@ -176,53 +187,59 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertTrue(triple.getMiddle() instanceof NDataModel);
Assert.assertNull(triple.getRight());
- val modelList2 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 5,
- "storage", true, null,
+ ModelQueryParams request2 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 5, "storage", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, true);
+ null, true, false);
+ val modelList2 = modelService.getModels(request2);
Assert.assertEquals(11, modelList2.getTotalSize());
Assert.assertEquals(5, modelList2.getValue().size());
Assert.assertTrue(((NDataModelResponse) modelList2.getValue().get(0))
.getStorage() >= ((NDataModelResponse) modelList2.getValue().get(4)).getStorage());
- val modelList3 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 10,
- "expansionrate", true, null,
+ ModelQueryParams request3 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 10, "expansionrate", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, false);
+ null, false, false);
+ val modelList3 = modelService.getModels(request3);
Assert.assertEquals(1, modelList3.getValue().size());
- val modelList4 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0, 10,
- "expansionrate", false, "ADMIN",
+ ModelQueryParams request4 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 0, 10, "expansionrate", false, "ADMIN",
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID),
- 974198646000L, System.currentTimeMillis(), true);
+ 974198646000L, System.currentTimeMillis(), true, false);
+ val modelList4 = modelService.getModels(request4);
Assert.assertEquals(10, modelList4.getValue().size());
- val modelList5 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 6, "",
- true, null,
+ ModelQueryParams request5 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 6, "", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, false);
+ null, false, false);
+ val modelList5 = modelService.getModels(request5);
Assert.assertEquals(5, modelList5.getValue().size());
getTestConfig().setProperty("kylin.metadata.semi-automatic-mode", "true");
- val modelList6 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 6, "",
- true, null, Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING,
+ ModelQueryParams request6 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 6, "", true, null, Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING,
ModelAttributeEnum.HYBRID, ModelAttributeEnum.SECOND_STORAGE),
- null, null, false);
+ null, null, false, false);
+ val modelList6 = modelService.getModels(request6);
Assert.assertEquals(5, modelList6.getValue().size());
getTestConfig().setProperty("kylin.metadata.semi-automatic-mode", "false");
// used for getModels without sortBy field
- val modelList7 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0, 6, "",
- true, null,
+ ModelQueryParams request7 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 0, 6, "", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, false);
+ null, false, false);
+ val modelList7 = modelService.getModels(request7);
Assert.assertEquals(6, modelList7.getValue().size());
}
@Test
public void testConvertToDataModelResponseBroken() {
List<ModelAttributeEnum> modelAttributeSet = Lists.newArrayList(ModelAttributeEnum.BATCH);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "", true,
- null, modelAttributeSet, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet, null, null, true, false);
val tripleList = modelQueryService.getModels(modelQueryParams);
ModelTriple modelTriple = tripleList.get(0);
@@ -257,17 +274,32 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertEquals(0, model1.getSource());
String modelName2 = "model_streaming";
- DataResult<List<NDataModel>> modelResult2 = modelService.getModels("4965c827-fbb4-4ea1-a744-3f341a3b030d",
- modelName2, true, project, "ADMIN", Lists.newArrayList(), "", 0, 10, "last_modify", true, null,
+ ModelQueryParams liteRequest = new ModelQueryParams("4965c827-fbb4-4ea1-a744-3f341a3b030d", modelName2, true,
+ project, "ADMIN", Lists.newArrayList(), "", 0, 10, "last_modify", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID,
ModelAttributeEnum.SECOND_STORAGE),
- null, null, true);
+ null, null, true, true);
+
+ DataResult<List<NDataModel>> modelResult2 = modelService.getModels(liteRequest);
List<NDataModel> models2 = modelResult2.getValue();
- FusionModelResponse model2 = (FusionModelResponse) models2.get(0);
+ NDataModelLiteResponse model2 = (NDataModelLiteResponse) models2.get(0);
Assert.assertEquals(12010, model2.getOldParams().getInputRecordCnt());
Assert.assertEquals(1505415, model2.getOldParams().getInputRecordSizeBytes());
Assert.assertEquals(396, model2.getOldParams().getSizeKB());
+
+ ModelQueryParams request = new ModelQueryParams("4965c827-fbb4-4ea1-a744-3f341a3b030d", modelName2, true,
+ project, "ADMIN", Lists.newArrayList(), "", 0, 10, "last_modify", true, null,
+ Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID,
+ ModelAttributeEnum.SECOND_STORAGE),
+ null, null, true, false);
+ DataResult<List<NDataModel>> modelResult3 = modelService.getModels(request);
+ List<NDataModel> models3 = modelResult3.getValue();
+ FusionModelResponse model3 = (FusionModelResponse) models3.get(0);
+
+ Assert.assertEquals(12010, model3.getOldParams().getInputRecordCnt());
+ Assert.assertEquals(1505415, model3.getOldParams().getInputRecordSizeBytes());
+ Assert.assertEquals(396, model3.getOldParams().getSizeKB());
}
@Test
@@ -279,9 +311,10 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertEquals(1, models2.size());
doReturn(new ArrayList<>()).when(modelService).addOldParams(anyString(), any());
- val models3 = modelService.getModels("741ca86a-1f13-46da-a59f-95fb68615e3a", null, true, "default", "ADMIN",
- Lists.newArrayList(), "DEFAULT.TEST_KYLIN_FACT", 0, 8, "last_modify", true, null, null, null, null,
- true);
+ ModelQueryParams request = new ModelQueryParams("741ca86a-1f13-46da-a59f-95fb68615e3a", null, true, "default",
+ "ADMIN", Lists.newArrayList(), "DEFAULT.TEST_KYLIN_FACT", 0, 8, "last_modify", true, null, null, null,
+ null, true, false);
+ val models3 = modelService.getModels(request);
Assert.assertEquals(1, models3.getTotalSize());
}
}
diff --git a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
index 006c03a3ee..67f2a7a35a 100644
--- a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
+++ b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
@@ -137,6 +137,7 @@ import org.apache.kylin.rest.service.QueryHistoryScheduler;
import org.apache.kylin.rest.service.QueryHistoryService;
import org.apache.kylin.rest.service.SegmentHelper;
import org.apache.kylin.rest.service.params.MergeSegmentParams;
+import org.apache.kylin.rest.service.params.ModelQueryParams;
import org.apache.kylin.rest.service.params.RefreshSegmentParams;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclUtil;
@@ -2088,8 +2089,9 @@ public class SecondStorageLockTest implements JobWaiter {
val sum = segments.stream().mapToLong(NDataSegmentResponse::getSecondStorageSize).sum();
- DataResult<List<NDataModel>> result = modelService.getModels(modelId, null, true, getProject(), null, null,
- null, 0, 10, "last_modify", false, null, null, null, null, true);
+ ModelQueryParams request = new ModelQueryParams(modelId, null, true, getProject(), null, null, null, 0, 10,
+ "last_modify", false, null, null, null, null, true, false);
+ DataResult<List<NDataModel>> result = modelService.getModels(request);
result.getValue().stream().filter(nDataModel -> modelId.equals(nDataModel.getId())).forEach(nDataModel -> {
val nDataModelRes = (NDataModelResponse) nDataModel;
[kylin] 25/38: KYLIN-5536 Limit the segment range of MAX query to improve query performance
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 5d251e313959b13cb15c2789eb69b6bb9b72c6b5
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Tue Feb 28 15:26:19 2023 +0800
KYLIN-5536 Limit the segment range of MAX query to improve query performance
Co-authored-by: sibing.zhang <si...@qq.com>
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +
.../apache/kylin/common/KylinConfigBaseTest.java | 9 ++
.../kylin/query/routing/RealizationPruner.java | 177 +++++++++++++++------
.../kylin/rest/service/QueryServiceTest.java | 16 +-
.../kylin/query/engine/SelectRealizationTest.java | 7 +-
5 files changed, 154 insertions(+), 59 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7f90613fad..19021c4d73 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3838,6 +3838,10 @@ public abstract class KylinConfigBase implements Serializable {
&& Boolean.parseBoolean(getOptional("kylin.query.diagnose-enabled", TRUE));
}
+ public long getMaxMeasureSegmentPrunerBeforeDays() {
+ return Long.parseLong(getOptional("kylin.query.max-measure-segment-pruner-before-days", "-1"));
+ }
+
// ============================================================================
// Cost based index Planner
// ============================================================================
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 5dfad30ca5..3bbd191aff 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -1427,6 +1427,15 @@ class KylinConfigBaseTest {
config.setProperty("kylin.query.calcite.bindable.cache.concurrencyLevel", "3");
assertEquals(3, config.getCalciteBindableCacheConcurrencyLevel());
}
+
+ @Test
+ void testGetMaxMeasureSegmentPrunerBeforeDays() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ long defaultValue = config.getMaxMeasureSegmentPrunerBeforeDays();
+ assertEquals(-1, defaultValue);
+ config.setProperty("kylin.query.max-measure-segment-pruner-before-days", "1");
+ assertEquals(1, config.getMaxMeasureSegmentPrunerBeforeDays());
+ }
}
class EnvironmentUpdateUtils {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationPruner.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationPruner.java
index 5c56de2fb3..905dcd02e1 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationPruner.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationPruner.java
@@ -60,6 +60,8 @@ import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
import org.apache.kylin.metadata.model.MultiPartitionKeyMapping;
@@ -67,6 +69,7 @@ import org.apache.kylin.metadata.model.MultiPartitionKeyMappingProvider;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -89,6 +92,7 @@ public class RealizationPruner {
private static final String STRING = "string";
private static final String INTEGER = "integer";
private static final String BIGINT = "bigint";
+ public static final long DAY = 24 * 3600 * 1000L;
private static final TimeZone UTC_ZONE = TimeZone.getTimeZone("UTC");
private static final Pattern DATE_PATTERN = Pattern.compile("[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]");
private static final Pattern TIMESTAMP_PATTERN = Pattern.compile(
@@ -102,80 +106,90 @@ public class RealizationPruner {
private RealizationPruner() {
}
- public static List<NDataSegment> pruneSegments(NDataflow dataflow, OLAPContext olapContext) {
- val kylinConfig = KylinConfig.getInstanceFromEnv();
- val projectName = dataflow.getProject();
-
- val projectInstance = NProjectManager.getInstance(kylinConfig).getProject(projectName);
- val allReadySegments = dataflow.getQueryableSegments();
- if (!projectInstance.getConfig().isHeterogeneousSegmentEnabled()) {
- return allReadySegments;
- }
- val isStreamingFactTable = olapContext.firstTableScan.getOlapTable().getSourceTable()
- .getSourceType() == ISourceAware.ID_STREAMING;
- val isBatchFusionModel = isStreamingFactTable && dataflow.getModel().isFusionModel() && !dataflow.isStreaming();
- val partitionDesc = isBatchFusionModel
- ? getStreamingPartitionDesc(dataflow.getModel(), kylinConfig, projectName)
- : dataflow.getModel().getPartitionDesc();
- // no partition column
- if (PartitionDesc.isEmptyPartitionDesc(partitionDesc)) {
- log.info("No partition column");
+ public static Segments<NDataSegment> pruneSegments(NDataflow dataflow, OLAPContext olapContext) {
+ Segments<NDataSegment> allReadySegments = dataflow.getQueryableSegments();
+ if (!NProjectManager.getProjectConfig(dataflow.getProject()).isHeterogeneousSegmentEnabled()) {
return allReadySegments;
}
- val partitionColumn = partitionDesc.getPartitionDateColumnRef();
- val dateFormat = partitionDesc.getPartitionDateFormat();
- val filterColumns = olapContext.filterColumns;
- // sql filter columns do not include partition column
- if (!filterColumns.contains(partitionColumn)) {
- log.info("Filter columns do not contain partition column");
+ // pruner segment by partition column and dataformat
+ PartitionDesc partitionCol = getPartitionDesc(dataflow, olapContext);
+ if (isFullBuildModel(partitionCol)) {
+ log.info("No partition column or partition column format is null.");
return allReadySegments;
}
- val selectedSegments = Lists.<NDataSegment> newArrayList();
- var filterConditions = olapContext.getExpandedFilterConditions();
+ // pruner segment by simplify sql filter
val relOptCluster = olapContext.firstTableScan.getCluster();
val rexBuilder = relOptCluster.getRexBuilder();
val rexSimplify = new RexSimplify(relOptCluster.getRexBuilder(), RelOptPredicateList.EMPTY, true,
relOptCluster.getPlanner().getExecutor());
- val partitionColInputRef = transformColumn2RexInputRef(partitionColumn, olapContext.allTableScans);
- if (allReadySegments.size() > 0 && dateFormat != null) {
+ var filterConditions = olapContext.getExpandedFilterConditions();
+ val dateFormat = partitionCol.getPartitionDateFormat();
+ val partitionColRef = partitionCol.getPartitionDateColumnRef();
+ RexInputRef partitionColInputRef = null;
+ if (needRewritePartitionColInFilter(dataflow, olapContext)) {
+ partitionColInputRef = transformColumn2RexInputRef(partitionColRef, olapContext.allTableScans);
try {
val firstSegmentRanges = transformSegment2RexCall(allReadySegments.get(0), dateFormat, rexBuilder,
- partitionColInputRef, partitionColumn.getType(), dataflow.isStreaming());
+ partitionColInputRef, partitionColRef.getType(), dataflow.isStreaming());
RelDataTypeFamily segmentLiteralTypeFamily = getSegmentLiteralTypeFamily(firstSegmentRanges.getFirst());
- filterConditions = filterConditions.stream()//
- .map(filterCondition -> rewriteRexCall(filterCondition, rexBuilder, segmentLiteralTypeFamily,
- partitionColInputRef, dateFormat))
- .collect(Collectors.toList());
+ List<RexNode> filterRexNodeList = new ArrayList<>();
+ for (RexNode filterCondition : filterConditions) {
+ RexNode rexNode = rewriteRexCall(filterCondition, rexBuilder, segmentLiteralTypeFamily,
+ partitionColInputRef, dateFormat);
+ filterRexNodeList.add(rexNode);
+ }
+ filterConditions = filterRexNodeList;
} catch (Exception ex) {
log.warn("Segment pruning error: ", ex);
+ if (canPruneSegmentsForMaxMeasure(dataflow, olapContext, partitionColRef)) {
+ return selectSegmentsForMaxMeasure(dataflow);
+ }
return allReadySegments;
}
}
- var simplifiedSqlFilter = rexSimplify.simplifyAnds(filterConditions);
- // sql filter condition is always false
+ RexNode simplifiedSqlFilter = rexSimplify.simplifyAnds(filterConditions);
if (simplifiedSqlFilter.isAlwaysFalse()) {
log.info("SQL filter condition is always false, pruning all ready segments");
olapContext.storageContext.setFilterCondAlwaysFalse(true);
- return selectedSegments;
+ return Segments.empty();
}
- // sql filter condition is always true
- if (simplifiedSqlFilter.isAlwaysTrue()) {
- log.info("SQL filter condition is always true, pruning no segment");
+
+ // pruner segment by customized scene optimize
+ if (canPruneSegmentsForMaxMeasure(dataflow, olapContext, partitionColRef)) {
+ return selectSegmentsForMaxMeasure(dataflow);
+ }
+
+ if (!olapContext.filterColumns.contains(partitionColRef)) {
+ log.info("Filter columns do not contain partition column");
return allReadySegments;
}
- if (dateFormat == null) {
+ if (simplifiedSqlFilter.isAlwaysTrue()) {
+ log.info("SQL filter condition is always true, pruning no segment");
return allReadySegments;
}
- for (NDataSegment dataSegment : allReadySegments) {
+ // prune segments by partition filter
+ Segments<NDataSegment> selectedSegments = pruneSegmentsByPartitionFilter(dataflow, olapContext, rexSimplify,
+ partitionColInputRef, simplifiedSqlFilter);
+ log.info("Scan segment.size: {} after segment pruning", selectedSegments.size());
+ return selectedSegments;
+ }
+
+ private static Segments<NDataSegment> pruneSegmentsByPartitionFilter(NDataflow dataflow, OLAPContext olapContext,
+ RexSimplify rexSimplify, RexInputRef partitionColInputRef, RexNode simplifiedSqlFilter) {
+ Segments<NDataSegment> selectedSegments = new Segments<>();
+ PartitionDesc partitionCol = getPartitionDesc(dataflow, olapContext);
+ RexBuilder rexBuilder = olapContext.firstTableScan.getCluster().getRexBuilder();
+ for (NDataSegment dataSegment : dataflow.getQueryableSegments()) {
try {
- val segmentRanges = transformSegment2RexCall(dataSegment, dateFormat, rexBuilder, partitionColInputRef,
- partitionColumn.getType(), dataflow.isStreaming());
+ val segmentRanges = transformSegment2RexCall(dataSegment, partitionCol.getPartitionDateFormat(),
+ rexBuilder, partitionColInputRef, partitionCol.getPartitionDateColumnRef().getType(),
+ dataflow.isStreaming());
// compare with segment start
val segmentStartPredicate = RelOptPredicateList.of(rexBuilder,
Lists.newArrayList(segmentRanges.getFirst()));
@@ -197,10 +211,79 @@ public class RealizationPruner {
selectedSegments.add(dataSegment);
}
}
- log.info("Scan segment.size: {} after segment pruning", selectedSegments.size());
return selectedSegments;
}
+ private static boolean needRewritePartitionColInFilter(NDataflow dataflow, OLAPContext olapContext) {
+ return !dataflow.getQueryableSegments().isEmpty() && olapContext.filterColumns
+ .contains(getPartitionDesc(dataflow, olapContext).getPartitionDateColumnRef());
+ }
+
+ private static boolean isFullBuildModel(PartitionDesc partitionCol) {
+ return PartitionDesc.isEmptyPartitionDesc(partitionCol) || partitionCol.getPartitionDateFormat() == null;
+ }
+
+ private static Segments<NDataSegment> selectSegmentsForMaxMeasure(NDataflow dataflow) {
+ Segments<NDataSegment> selectedSegments = new Segments<>();
+ long days = dataflow.getConfig().getMaxMeasureSegmentPrunerBeforeDays();
+ // segment was sorted
+ Segments<NDataSegment> allReadySegments = dataflow.getQueryableSegments();
+ long maxDt = allReadySegments.getLatestReadySegment().getTSRange().getEnd();
+ long minDt = maxDt - DAY * days;
+ for (int i = allReadySegments.size() - 1; i >= 0; i--) {
+ if (allReadySegments.get(i).getTSRange().getEnd() > minDt) {
+ selectedSegments.add(allReadySegments.get(i));
+ } else {
+ break;
+ }
+ }
+ log.info("Scan segment size: {} after max measure segment pruner. The before days: {}. Passed on segment: {}",
+ selectedSegments.size(), days,
+ selectedSegments.stream().map(ISegment::getName).collect(Collectors.joining(",")));
+ return selectedSegments;
+ }
+
+ private static boolean canPruneSegmentsForMaxMeasure(NDataflow dataflow, OLAPContext olapContext,
+ TblColRef partitionColRef) {
+ if (dataflow.getConfig().getMaxMeasureSegmentPrunerBeforeDays() < 0) {
+ return false;
+ }
+
+ if (CollectionUtils.isNotEmpty(olapContext.getGroupByColumns())
+ && !olapContext.getGroupByColumns().stream().allMatch(partitionColRef::equals)) {
+ return false;
+ }
+
+ if (CollectionUtils.isEmpty(olapContext.aggregations)) {
+ return false;
+ }
+
+ for (FunctionDesc agg : olapContext.aggregations) {
+ if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(agg.getExpression())
+ && !partitionColRef.equals(agg.getParameters().get(0).getColRef())) {
+ return false;
+ }
+ if (!FunctionDesc.FUNC_MAX.equalsIgnoreCase(agg.getExpression())
+ && CollectionUtils.isNotEmpty(agg.getParameters())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private static PartitionDesc getPartitionDesc(NDataflow dataflow, OLAPContext olapContext) {
+ NDataModel model = dataflow.getModel();
+ val isStreamingFactTable = olapContext.firstTableScan.getOlapTable().getSourceTable()
+ .getSourceType() == ISourceAware.ID_STREAMING;
+ val isBatchFusionModel = isStreamingFactTable && dataflow.getModel().isFusionModel() && !dataflow.isStreaming();
+ if (!isBatchFusionModel) {
+ return model.getPartitionDesc();
+ }
+ return NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), dataflow.getProject())
+ .getDataModelDesc(model.getFusionId()).getPartitionDesc();
+ }
+
public static RexNode rewriteRexCall(RexNode rexNode, RexBuilder rexBuilder, RelDataTypeFamily relDataTypeFamily,
RexInputRef partitionColInputRef, String dateFormat) {
if (!(rexNode instanceof RexCall)) {
@@ -306,12 +389,6 @@ public class RealizationPruner {
return null;
}
- private static PartitionDesc getStreamingPartitionDesc(NDataModel model, KylinConfig kylinConfig, String project) {
- NDataModelManager modelManager = NDataModelManager.getInstance(kylinConfig, project);
- NDataModel streamingModel = modelManager.getDataModelDesc(model.getFusionId());
- return streamingModel.getPartitionDesc();
- }
-
private static Pair<RexNode, RexNode> transformSegment2RexCall(NDataSegment dataSegment, String dateFormat,
RexBuilder rexBuilder, RexInputRef partitionColInputRef, DataType partitionColType, boolean isStreaming) {
String start;
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 4f1c9c1fba..46a0ca4b2c 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -2104,17 +2104,17 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
PrepareSqlStateParam[] params1 = new PrepareSqlStateParam[2];
params1[0] = new PrepareSqlStateParam(Double.class.getCanonicalName(), "123.1");
params1[1] = new PrepareSqlStateParam(Integer.class.getCanonicalName(), "123");
- String originSql1 = "with t1 as (select ORDER_ID, PRICE > ? from test_kylin_fact where ORDER_ID = ?)\n, "
- + "t2 as (select bf from test_kylin_fact)\n" + "select * from t1 where ORDER_ID = '100'\n" //
+ String originSql1 = "with t1 as (select ORDER_ID, PRICE > ? from test_kylin_fact where ORDER_ID > ?)\n, "
+ + "t2 as (select bf from test_kylin_fact)\n" + "select * from t1 where ORDER_ID = '125'\n" //
+ "union all\n" //
- + "select * from t1 where ORDER_ID = '200'";
- String filledSql1 = "with t1 as (select ORDER_ID, PRICE > 123.1 from test_kylin_fact where ORDER_ID = 123)\n"
- + ", t2 as (select bf from test_kylin_fact)\n" + "select * from t1 where ORDER_ID = '100'\n"
- + "union all\n" + "select * from t1 where ORDER_ID = '200'";
+ + "select * from t1 where ORDER_ID < '200'";
+ String filledSql1 = "with t1 as (select ORDER_ID, PRICE > 123.1 from test_kylin_fact where ORDER_ID > 123)\n"
+ + ", t2 as (select bf from test_kylin_fact)\n" + "select * from t1 where ORDER_ID = '125'\n"
+ + "union all\n" + "select * from t1 where ORDER_ID < '200'";
String transformedFilledSql1 = "SELECT *\n" + "FROM (SELECT ORDER_ID, PRICE > 123.1\n"
- + "FROM TEST_KYLIN_FACT\n" + "WHERE ORDER_ID = 123) AS T1\n" + "WHERE ORDER_ID = '100'\n"
+ + "FROM TEST_KYLIN_FACT\n" + "WHERE ORDER_ID > 123) AS T1\n" + "WHERE ORDER_ID = '125'\n"
+ "UNION ALL\n" + "SELECT *\n" + "FROM (SELECT ORDER_ID, PRICE > 123.1\n" + "FROM TEST_KYLIN_FACT\n"
- + "WHERE ORDER_ID = 123) AS T1\n" + "WHERE ORDER_ID = '200'";
+ + "WHERE ORDER_ID > 123) AS T1\n" + "WHERE ORDER_ID < '200'";
queryWithParamWhenTransformWithToSubQuery(params1, originSql1, filledSql1, transformedFilledSql1);
PrepareSqlStateParam[] params2 = new PrepareSqlStateParam[1];
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java
index f5ab4fbe1b..453ae67d60 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java
@@ -31,11 +31,13 @@ import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.query.calcite.KylinRelDataTypeSystem;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.query.QueryExtension;
+import org.apache.kylin.query.calcite.KylinRelDataTypeSystem;
+import org.apache.kylin.query.engine.meta.SimpleDataContext;
import org.apache.kylin.query.util.QueryContextCutter;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -68,6 +70,9 @@ class SelectRealizationTest {
String defaultSchemaName = schemaFactory.getDefaultSchema();
val catalogReader = createCatalogReader(config, rootSchema, defaultSchemaName);
val planner = new PlannerFactory(kylinConfig).createVolcanoPlanner(config);
+ SimpleDataContext dataContext = new SimpleDataContext(rootSchema.plus(), TypeSystem.javaTypeFactory(),
+ kylinConfig);
+ planner.setExecutor(new RexExecutorImpl(dataContext));
val sqlConverter = SQLConverter.createConverter(config, planner, catalogReader);
val queryOptimizer = new QueryOptimizer(planner);
RelRoot relRoot = sqlConverter
[kylin] 37/38: mirror: remove unused import
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f508aa3439f6a66c9cf9e7b354e7055b296d9a41
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Wed Apr 26 10:30:44 2023 +0800
mirror: remove unused import
---
.../org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
index 6131b84e70..abb8abc371 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.service;
import lombok.extern.slf4j.Slf4j;
-import com.google.common.collect.ImmutableList;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.engine.spark.utils.SparkJobFactoryUtils;
import org.apache.kylin.junit.rule.TransactionExceptedException;
[kylin] 03/38: KYLIN-5522 fix local debug mode
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2cf578b0fa0c93292597e291f11ccead7c217762
Author: Junqing Cai <ca...@163.com>
AuthorDate: Wed Feb 8 17:56:27 2023 +0800
KYLIN-5522 fix local debug mode
* use commons-lang instead of commons-lang3
* fix local debug mode
---
.../kylin/rest/controller/NBasicController.java | 2 +-
.../apache/kylin/rest/KylinPrepareEnvListener.java | 2 +-
.../kylin/rest/aspect/InsensitiveNameAspect.java | 2 +-
.../apache/kylin/rest/broadcaster/Broadcaster.java | 2 +-
.../rest/config/initialize/AclTCRListener.java | 2 +-
.../rest/config/initialize/CacheCleanListener.java | 2 +-
.../rest/config/initialize/MetricsRegistry.java | 2 +-
.../initialize/TableSchemaChangeListener.java | 2 +-
.../ResourceGroupEntityValidator.java | 2 +-
.../ResourceGroupKylinInstanceValidator.java | 2 +-
.../ResourceGroupMappingInfoValidator.java | 2 +-
.../interceptor/RepeatableRequestBodyFilter.java | 2 +-
.../interceptor/ResourceGroupCheckerFilter.java | 2 +-
.../apache/kylin/rest/monitor/MonitorReporter.java | 2 +-
.../org/apache/kylin/rest/request/UserRequest.java | 2 +-
.../apache/kylin/rest/service/AccessService.java | 2 +-
.../kylin/rest/service/AsyncTaskService.java | 2 +-
.../apache/kylin/rest/service/BasicService.java | 2 +-
.../apache/kylin/rest/service/LdapUserService.java | 2 +-
.../rest/service/MockQuerySmartSupporter.java} | 31 ++++++----------------
.../kylin/rest/service/OpenUserGroupService.java | 2 +-
.../apache/kylin/rest/service/ProjectService.java | 2 +-
.../apache/kylin/rest/service/SystemService.java | 2 +-
.../apache/kylin/rest/service/UserAclService.java | 2 +-
.../java/org/apache/kylin/tool/util/ToolUtil.java | 2 +-
.../java/org/apache/kylin/common/KylinConfig.java | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../java/org/apache/kylin/common/KylinVersion.java | 2 +-
.../java/org/apache/kylin/common/StorageURL.java | 2 +-
.../apache/kylin/common/debug/BackdoorToggles.java | 2 +-
.../kylin/common/persistence/StringEntity.java | 2 +-
.../persistence/metadata/JdbcMetadataStore.java | 2 +-
.../transaction/AuditLogReplayWorker.java | 2 +-
.../kylin/common/util/DaemonThreadFactory.java | 2 +-
.../org/apache/kylin/common/util/DateFormat.java | 2 +-
.../org/apache/kylin/common/util/EncryptUtil.java | 2 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 2 +-
.../kylin/common/util/OrderedProperties.java | 2 +-
.../org/apache/kylin/common/util/SSHClient.java | 2 +-
.../java/org/apache/kylin/common/util/Unsafe.java | 2 +-
.../org/apache/kylin/common/util/ZipFileUtils.java | 2 +-
.../org/apache/kylin/rest/util/PagingUtil.java | 2 +-
.../common/util/NLocalFileMetadataTestCase.java | 2 +-
.../apache/kylin/engine/spark/ExecutableUtils.java | 2 +-
.../org/apache/kylin/job/dao/ExecutablePO.java | 2 +-
.../apache/kylin/job/engine/JobEngineConfig.java | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 2 +-
.../job/execution/DefaultExecutableOnModel.java | 2 +-
.../kylin/job/execution/ExecutableParams.java | 2 +-
.../job/execution/NExecutableManagerTest.java | 2 +-
.../job/impl/threadpool/NDefaultSchedulerTest.java | 2 +-
.../apache/kylin/metadata/acl/AclTCRManager.java | 2 +-
.../metadata/cube/cuboid/AggIndexMatcher.java | 2 +-
.../kylin/metadata/cube/cuboid/IndexMatcher.java | 2 +-
.../metadata/cube/cuboid/KECuboidSchedulerV1.java | 2 +-
.../cube/model/NDataLoadingRangeManager.java | 2 +-
.../kylin/metadata/cube/model/NDataSegment.java | 2 +-
.../cube/model/NDataflowCapabilityChecker.java | 2 +-
.../metadata/cube/model/NIndexPlanManager.java | 2 +-
.../cube/storage/ProjectStorageInfoCollector.java | 2 +-
.../kylin/metadata/datatype/BooleanSerializer.java | 2 +-
.../apache/kylin/metadata/datatype/DataType.java | 2 +-
.../apache/kylin/metadata/epoch/EpochManager.java | 2 +-
.../metadata/filter/function/LikeMatchers.java | 2 +-
.../insensitive/ModelInsensitiveRequest.java | 2 +-
.../insensitive/ProjectInsensitiveRequest.java | 2 +-
.../insensitive/UserInsensitiveRequest.java | 2 +-
.../kylin/metadata/model/ComputedColumnDesc.java | 2 +-
.../kylin/metadata/model/JoinedFlatTable.java | 2 +-
.../apache/kylin/metadata/model/MeasureDesc.java | 2 +-
.../metadata/model/ModelJoinRelationTypeEnum.java | 2 +-
.../kylin/metadata/model/NDataModelManager.java | 2 +-
.../metadata/model/NTableMetadataManager.java | 2 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 2 +-
.../metadata/model/alias/ExpressionComparator.java | 2 +-
.../model/schema/AffectedModelContext.java | 2 +-
.../metadata/model/schema/ModelEdgeCollector.java | 2 +-
.../kylin/metadata/model/tool/CalciteParser.java | 2 +-
.../metadata/query/JdbcQueryHistoryStore.java | 2 +-
.../kylin/metadata/realization/SQLDigest.java | 2 +-
.../recommendation/candidate/RawRecItemTable.java | 2 +-
.../recommendation/entity/CCRecItemV2.java | 2 +-
.../metadata/recommendation/ref/OptRecV2.java | 2 +-
.../metadata/sourceusage/SourceUsageManager.java | 2 +-
.../kylin/metadata/streaming/KafkaConfig.java | 2 +-
.../kylin/rest/security/ExternalAclProvider.java | 2 +-
.../apache/kylin/rest/security/UserAclManager.java | 2 +-
.../metadata/epoch/EpochUpdateLockManagerTest.java | 2 +-
.../apache/kylin/common/metrics/MetricsGroup.java | 2 +-
.../common/metrics/MetricsInfluxdbReporter.java | 2 +-
.../kylin/rest/controller/BaseController.java | 2 +-
.../kylin/rest/controller/JobController.java | 2 +-
.../kylin/rest/controller/SegmentController.java | 2 +-
.../open/OpenStreamingJobController.java | 2 +-
.../rest/controller/v2/SegmentControllerV2.java | 2 +-
.../rest/controller/SampleControllerTest.java | 2 +-
.../controller/StreamingJobControllerTest.java | 2 +-
.../controller/open/OpenSampleControllerTest.java | 2 +-
.../open/OpenStreamingJobControllerTest.java | 2 +-
.../kylin/rest/response/ExecutableResponse.java | 2 +-
.../org/apache/kylin/rest/service/JobService.java | 2 +-
.../kylin/rest/service/ModelBuildService.java | 2 +-
.../apache/kylin/rest/service/SnapshotService.java | 2 +-
.../kylin/rest/service/TableSamplingService.java | 2 +-
.../apache/kylin/rest/service/JobServiceTest.java | 2 +-
.../sdk/datasource/adaptor/DefaultAdaptor.java | 2 +-
.../sdk/datasource/framework/JdbcConnector.java | 2 +-
.../datasource/framework/conv/ParamNodeParser.java | 2 +-
.../datasource/framework/conv/SqlConverter.java | 2 +-
.../apache/kylin/rest/service/SparkDDLService.java | 2 +-
.../apache/kylin/rest/service/TableService.java | 2 +-
.../rest/config/initialize/JobSyncListener.java | 2 +-
.../rest/config/initialize/JobSchedulerTest.java | 2 +-
.../java/org/apache/kylin/query/NKapQueryTest.java | 2 +-
.../rest/controller/NProjectControllerTest.java | 2 +-
.../kylin/rest/controller/NModelController.java | 2 +-
.../kylin/rest/controller/NProjectController.java | 2 +-
.../kylin/rest/controller/NTableController.java | 4 +--
.../kylin/rest/controller/NUserController.java | 2 +-
.../rest/controller/NUserGroupController.java | 2 +-
.../rest/controller/v2/NAccessControllerV2.java | 2 +-
.../rest/controller/NTableControllerTest.java | 2 +-
.../kylin/rest/controller/NUserControllerTest.java | 2 +-
.../apache/kylin/rest/model/FuzzyKeySearcher.java | 2 +-
.../response/SynchronizedCommentsResponse.java | 2 +-
.../kylin/rest/service/AbstractModelService.java | 2 +-
.../kylin/rest/service/FusionModelService.java | 2 +-
.../kylin/rest/service/IndexPlanService.java | 2 +-
.../kylin/rest/service/ModelSemanticHelper.java | 2 +-
.../apache/kylin/rest/service/ModelService.java | 4 +--
.../org/apache/kylin/rest/util/ModelUtils.java | 2 +-
.../apache/kylin/rest/service/BaseIndexTest.java | 2 +-
.../kylin/rest/service/FusionModelServiceTest.java | 2 +-
.../kylin/rest/service/ModelServiceTest.java | 4 +--
.../kylin/rest/service/TableServiceTest.java | 2 +-
.../kylin/query/routing/RealizationChooser.java | 2 +-
.../org/apache/kylin/query/schema/OLAPTable.java | 2 +-
.../query/security/AccessDeniedException.java | 2 +-
.../org/apache/kylin/query/security/RowFilter.java | 2 +-
.../apache/kylin/query/util/AsyncQueryUtil.java | 2 +-
.../query/util/ModelViewSqlNodeComparator.java | 2 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 2 +-
.../apache/kylin/query/util/QueryAliasMatcher.java | 2 +-
.../rest/controller/NAsyncQueryController.java | 2 +-
.../kylin/rest/controller/NQueryController.java | 2 +-
.../controller/SparkMetricsControllerTest.java | 2 +-
.../apache/kylin/rest/response/SQLResponseV2.java | 2 +-
.../kylin/rest/service/ModelQueryService.java | 2 +-
.../apache/kylin/rest/service/MonitorService.java | 2 +-
.../kylin/rest/service/QueryHistoryService.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 2 +-
.../kylin/rest/util/QueryCacheSignatureUtil.java | 2 +-
.../org/apache/kylin/rest/util/QueryUtils.java | 2 +-
.../kylin/query/engine/AsyncQueryApplication.java | 2 +-
.../apache/kylin/query/engine/AsyncQueryJob.java | 2 +-
.../kylin/query/engine/view/ModelViewTest.java | 2 +-
.../kylin/query/schema/KylinSqlValidatorTest.java | 2 +-
.../kap/clickhouse/ClickHouseStorage.java | 2 +-
.../kyligence/kap/clickhouse/job/ClickHouse.java | 15 +++++------
.../kap/clickhouse/job/ClickHouseLoad.java | 2 +-
.../kap/clickhouse/job/ClickhouseLoadFileLoad.java | 2 +-
.../kap/clickhouse/job/HadoopMockUtil.java | 2 +-
.../management/OpenSecondStorageEndpoint.java | 2 +-
.../management/SecondStorageEndpoint.java | 2 +-
.../io/kyligence/kap/secondstorage/ddl/DDL.java | 2 +-
.../kap/secondstorage/ddl/exp/TableIdentifier.java | 2 +-
.../kap/secondstorage/metadata/Manager.java | 2 +-
.../org/apache/kylin/rest/QueryNodeFilter.java | 2 +-
.../apache/kylin/rest/ZookeeperClusterManager.java | 2 +-
.../org/apache/kylin/rest/config/CorsConfig.java | 2 +-
.../rest/discovery/KylinServiceDiscoveryCache.java | 4 +--
.../discovery/KylinServiceDiscoveryClient.java | 2 +-
.../source/hive/BeelineOptionsProcessorTest.java | 2 +-
.../engine/spark/application/SparkApplication.java | 2 +-
.../spark/job/DefaultSparkBuildJobHandler.java | 2 +-
.../spark/job/ExecutableAddCuboidHandler.java | 2 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 2 +-
.../job/SparkCleanupTransactionalTableStep.java | 2 +-
.../kylin/engine/spark/mockup/CsvSource.java | 2 +-
.../kylin/engine/spark/mockup/CsvTableReader.java | 2 +-
.../engine/spark/source/NSparkDataSource.java | 2 +-
.../spark/source/NSparkMetadataExplorer.java | 2 +-
.../spark/stats/analyzer/TableAnalyzerJob.java | 2 +-
.../spark/stats/utils/DateTimeCheckUtils.java | 2 +-
.../spark/utils/HiveTransactionTableHelper.java | 2 +-
.../engine/spark/utils/SparkJobFactoryUtils.java | 2 +-
.../kylin/engine/spark/job/SegmentBuildJob.java | 2 +-
.../kylin/engine/spark/job/SegmentMergeJob.java | 2 +-
.../kylin/engine/spark/job/SnapshotBuildJob.java | 2 +-
.../org/apache/kylin/source/jdbc/JdbcSource.java | 2 +-
.../kylin/engine/spark/job/NSparkCubingUtil.java | 2 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java | 2 +-
.../asyncprofiler/AsyncProfilerToolTest.java | 2 +-
.../kylin/rest/service/StreamingJobService.java | 2 +-
.../org/apache/kylin/kafka/util/KafkaUtils.java | 2 +-
.../streaming/jobs/AbstractSparkJobLauncher.java | 2 +-
.../streaming/manager/StreamingJobManager.java | 2 +-
.../apache/kylin/streaming/rest/RestSupport.java | 2 +-
.../kylin/tool/AbstractInfoExtractorTool.java | 2 +-
.../java/org/apache/kylin/tool/AuditLogTool.java | 2 +-
.../org/apache/kylin/tool/ClickhouseDiagTool.java | 4 +--
.../main/java/org/apache/kylin/tool/ConfTool.java | 2 +-
.../apache/kylin/tool/DumpHadoopSystemProps.java | 2 +-
.../org/apache/kylin/tool/JobDiagInfoTool.java | 2 +-
.../java/org/apache/kylin/tool/KylinLogTool.java | 2 +-
.../org/apache/kylin/tool/daemon/KapGuardian.java | 2 +-
.../apache/kylin/tool/obf/KylinConfObfuscator.java | 2 +-
.../kylin/tool/upgrade/CheckProjectModeCLI.java | 2 +-
.../kylin/tool/upgrade/DeleteFavoriteQueryCLI.java | 2 +-
.../apache/kylin/tool/upgrade/RenameEntity.java | 2 +-
.../kylin/tool/upgrade/RenameUserResourceTool.java | 2 +-
.../apache/kylin/tool/upgrade/UpdateModelCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateProjectCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateSessionTableCLI.java | 2 +-
.../kylin/tool/upgrade/UpdateUserGroupCLI.java | 2 +-
.../kylin/tool/util/HadoopConfExtractor.java | 2 +-
.../org/apache/kylin/tool/util/ServerInfoUtil.java | 2 +-
.../org/apache/kylin/tool/StorageCleanerTest.java | 2 +-
.../kylin/tool/garbage/SnapshotCleanerTest.java | 2 +-
.../apache/kylin/tool/general/CryptToolTest.java | 2 +-
.../tool/upgrade/RenameUserResourceToolTest.java | 2 +-
221 files changed, 239 insertions(+), 255 deletions(-)
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index 914338294a..4115fd488b 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -71,7 +71,7 @@ import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index 483acca4b9..5a83565d0c 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -22,7 +22,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.TempMetadataBuilder;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
index d4631a785a..1d05598f3a 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.aspect;
import java.util.Objects;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.project.ProjectInstance;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java b/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java
index 92b9686557..d517a82742 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/broadcaster/Broadcaster.java
@@ -38,7 +38,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.constant.LogConstant;
import org.apache.kylin.common.logging.SetLogCategory;
import org.apache.kylin.common.persistence.transaction.BroadcastEventReadyNotifier;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/AclTCRListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/AclTCRListener.java
index 33d93145e8..9a2dc2a6d0 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/AclTCRListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/AclTCRListener.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.config.initialize;
import java.util.Objects;
import java.util.Optional;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.transaction.EventListenerRegistry;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/CacheCleanListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/CacheCleanListener.java
index 175b62970e..23b7aa3a37 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/CacheCleanListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/CacheCleanListener.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java
index 987df10e52..2ded66dbab 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/MetricsRegistry.java
@@ -32,7 +32,7 @@ import java.util.stream.Stream;
import javax.sql.DataSource;
import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.event.ModelAddEvent;
import org.apache.kylin.common.metrics.MetricsCategory;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/TableSchemaChangeListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/TableSchemaChangeListener.java
index c0a0ae3950..05469db9fd 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/TableSchemaChangeListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/TableSchemaChangeListener.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.config.initialize;
import java.util.Objects;
import java.util.Optional;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.transaction.EventListenerRegistry;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupEntityValidator.java b/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupEntityValidator.java
index 06e791f7ae..9670ccbd44 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupEntityValidator.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupEntityValidator.java
@@ -24,7 +24,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.RESOURCE_GR
import java.util.HashMap;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.resourcegroup.ResourceGroupEntity;
import org.apache.kylin.rest.request.resourecegroup.ResourceGroupRequest;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupKylinInstanceValidator.java b/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupKylinInstanceValidator.java
index 794093f478..f9f6f83824 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupKylinInstanceValidator.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupKylinInstanceValidator.java
@@ -25,7 +25,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.RESOURCE_GR
import java.util.List;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.resourcegroup.KylinInstance;
import org.apache.kylin.metadata.resourcegroup.ResourceGroupEntity;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupMappingInfoValidator.java b/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupMappingInfoValidator.java
index 8dc047e262..f6ccb87c55 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupMappingInfoValidator.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/handler/resourcegroup/ResourceGroupMappingInfoValidator.java
@@ -27,7 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.project.ProjectInstance;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/RepeatableRequestBodyFilter.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/RepeatableRequestBodyFilter.java
index 1d8db53c62..cc6d778d81 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/RepeatableRequestBodyFilter.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/RepeatableRequestBodyFilter.java
@@ -28,7 +28,7 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.Pair;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ResourceGroupCheckerFilter.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ResourceGroupCheckerFilter.java
index 07e7e72587..7b30f0e66a 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ResourceGroupCheckerFilter.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ResourceGroupCheckerFilter.java
@@ -31,7 +31,7 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/monitor/MonitorReporter.java b/src/common-service/src/main/java/org/apache/kylin/rest/monitor/MonitorReporter.java
index 16bfc47612..eb893e2216 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/monitor/MonitorReporter.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/monitor/MonitorReporter.java
@@ -26,7 +26,7 @@ import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.util.ExecutorServiceUtil;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/request/UserRequest.java b/src/common-service/src/main/java/org/apache/kylin/rest/request/UserRequest.java
index 1ba4546776..a29b6ff8c1 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/request/UserRequest.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/request/UserRequest.java
@@ -26,7 +26,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ArgsTypeJsonDeserializer;
import org.apache.kylin.metadata.insensitive.UserInsensitiveRequest;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 41230847eb..40ebeb23f3 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -63,7 +63,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
index 1b26150623..a343a464c6 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
@@ -32,7 +32,7 @@ import java.util.stream.Collectors;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.metrics.MetricsCategory;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
index cfbf3354e9..aa245ac9ee 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -28,7 +28,7 @@ import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.BroadcastEventReadyNotifier;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/LdapUserService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/LdapUserService.java
index 89fdae3ac9..5aac3f2f17 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/LdapUserService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/LdapUserService.java
@@ -36,7 +36,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.naming.directory.SearchControls;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.CaseInsensitiveStringMap;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/MockQuerySmartSupporter.java
similarity index 55%
copy from src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
copy to src/common-service/src/main/java/org/apache/kylin/rest/service/MockQuerySmartSupporter.java
index 5c4b0444fb..4ec15c5557 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/MockQuerySmartSupporter.java
@@ -15,32 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.common.util;
+package org.apache.kylin.rest.service;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
+import java.util.List;
-import org.apache.commons.lang.StringUtils;
-
-/**
- */
-public class DaemonThreadFactory implements ThreadFactory {
- private String poolName = null;
-
- public DaemonThreadFactory() {
- }
-
- public DaemonThreadFactory(String poolName) {
- this.poolName = poolName;
- }
+import org.apache.kylin.metadata.query.QueryHistory;
+import org.springframework.stereotype.Component;
+@Component
+public class MockQuerySmartSupporter implements QuerySmartSupporter {
@Override
- public Thread newThread(Runnable r) {
- Thread t = Executors.defaultThreadFactory().newThread(r);
- if (StringUtils.isNotBlank(poolName)) {
- t.setName(poolName.concat("-") + t.getId());
- }
- t.setDaemon(true);
- return t;
+ public void onMatchQueryHistory(String project, List<QueryHistory> queries, boolean manual) {
+ //do nothing
}
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/OpenUserGroupService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/OpenUserGroupService.java
index d26e91e85d..55c92daad6 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/OpenUserGroupService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/OpenUserGroupService.java
@@ -23,7 +23,7 @@ import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.annotation.ThirdPartyDependencies;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.metadata.user.ManagedUser;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 1af9955759..9d4663c301 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -59,7 +59,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigBase;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
index 63ddcda042..fa467e414d 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
@@ -42,7 +42,7 @@ import java.util.stream.Collectors;
import javax.validation.constraints.NotNull;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java
index 90641b3154..4ce2a58b9e 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java
@@ -33,7 +33,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
index b4cccb2868..f19d575bab 100644
--- a/src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
@@ -31,7 +31,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 9e542921e5..a0908f7043 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -40,7 +40,7 @@ import javax.annotation.Nullable;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.OrderedProperties;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b1f8541092..f1770e9fb2 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -52,7 +52,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 48461e155e..0230eb19ac 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/StorageURL.java b/src/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
index 46c58aa489..d034bff18c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/StorageURL.java
@@ -23,7 +23,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 4a4a2ca2ce..f69f0aab4c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -20,7 +20,7 @@ package org.apache.kylin.common.debug;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Maps;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java
index 71ef48f8db..4c3113ed24 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/StringEntity.java
@@ -22,7 +22,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.Setter;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
index 092c80c334..ad775a0334 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
@@ -36,7 +36,7 @@ import javax.annotation.Nullable;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.RawResource;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/AuditLogReplayWorker.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/AuditLogReplayWorker.java
index 63299c0aa4..380729ab4c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/AuditLogReplayWorker.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/AuditLogReplayWorker.java
@@ -31,7 +31,7 @@ import java.util.stream.Collectors;
import java.util.stream.LongStream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.LogConstant;
import org.apache.kylin.common.logging.SetLogCategory;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java b/src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
index 5c4b0444fb..4b7a391105 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
@@ -20,7 +20,7 @@ package org.apache.kylin.common.util;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
/**
*/
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index c70edf46d2..d7c541ed41 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -34,7 +34,7 @@ import java.util.Map;
import java.util.TimeZone;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
index c308754f12..ba73b9de0f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
@@ -24,7 +24,7 @@ import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public class EncryptUtil {
/**
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 5137e9ead0..8e4250fc0f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -32,7 +32,7 @@ import java.util.Locale;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java b/src/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java
index 8ed35b42cd..839bca8a14 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/OrderedProperties.java
@@ -43,7 +43,7 @@ import java.util.Vector;
import javax.annotation.Nonnull;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Preconditions;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java b/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
index cf3afcb227..a93363c0ff 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
@@ -31,7 +31,7 @@ import java.io.OutputStream;
import java.nio.charset.Charset;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.Channel;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
index 2fbf08fce1..4de01d989b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
@@ -23,7 +23,7 @@ import java.text.MessageFormat;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.SystemPropertiesCache;
/**
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java b/src/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
index b4c402be99..fb9394c6c8 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
@@ -31,7 +31,7 @@ import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.CommonErrorCode;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java b/src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java
index 6ae7c40c25..613de19dc7 100644
--- a/src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/rest/util/PagingUtil.java
@@ -23,7 +23,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public class PagingUtil {
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java b/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java
index f677ab9a49..212dea3c0c 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/util/NLocalFileMetadataTestCase.java
@@ -27,7 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.AbstractTestCase;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
diff --git a/src/core-job/src/main/java/org/apache/kylin/engine/spark/ExecutableUtils.java b/src/core-job/src/main/java/org/apache/kylin/engine/spark/ExecutableUtils.java
index 11636e770f..5629b167b8 100644
--- a/src/core-job/src/main/java/org/apache/kylin/engine/spark/ExecutableUtils.java
+++ b/src/core-job/src/main/java/org/apache/kylin/engine/spark/ExecutableUtils.java
@@ -23,7 +23,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.job.execution.AbstractExecutable;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
index 2c8c62a7a0..cfa3da6e19 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/ExecutablePO.java
@@ -26,7 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.job.constant.ExecutableConstants;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/src/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 0a3f882320..b08539eb87 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Locale;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 2baf95144c..2805ee0b31 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -52,7 +52,7 @@ import java.util.stream.Collectors;
import io.kyligence.kap.guava20.shaded.common.base.Throwables;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
index a87887f492..339efeef8d 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.job.model.JobParam;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableParams.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableParams.java
index bdd07a3201..2fa462efc2 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableParams.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableParams.java
@@ -35,7 +35,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MailHelper;
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java b/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
index 8934603053..4ba31b27c1 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/execution/NExecutableManagerTest.java
@@ -43,7 +43,7 @@ import java.util.Map;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java b/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java
index 1ed40dbbd8..4f56118395 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/impl/threadpool/NDefaultSchedulerTest.java
@@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
index ccd40959fc..fe8078bcad 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
@@ -32,7 +32,7 @@ import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.text.StrBuilder;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
index 7854f7a8f1..1fb13bc80b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
@@ -28,7 +28,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.cube.model.IndexEntity;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java
index 8dad4df87e..d485f8524a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java
@@ -26,7 +26,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataflow;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/KECuboidSchedulerV1.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/KECuboidSchedulerV1.java
index 325898a158..ed5a851a6b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/KECuboidSchedulerV1.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/KECuboidSchedulerV1.java
@@ -32,7 +32,7 @@ import java.util.stream.Stream;
import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.OutOfMaxCombinationException;
import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.util.ThreadUtil;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
index 28f220dd6c..5ffeb2e579 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
index b0e4795c6d..ddb81e407a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
@@ -30,7 +30,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.metadata.model.ISegment;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java
index 34d3a016f8..82e1e82853 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
index be1a135a13..223fdb2f4e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
index 1148d304e3..8d2dc7f03b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.cube.storage;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import com.google.common.collect.Lists;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
index 0920f831f8..b86d729cb4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.datatype;
import java.nio.ByteBuffer;
import java.util.Locale;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang.BooleanUtils;
public class BooleanSerializer extends DataTypeSerializer<Long> {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index ce7081c1bc..81e5881ef5 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -33,7 +33,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/epoch/EpochManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/epoch/EpochManager.java
index a68d25eeec..c7b22204e3 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/epoch/EpochManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/epoch/EpochManager.java
@@ -42,7 +42,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.constant.LogConstant;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java
index 9ecbf0762a..2126a22175 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/LikeMatchers.java
@@ -20,7 +20,7 @@ package org.apache.kylin.metadata.filter.function;
import java.util.Locale;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Preconditions;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ModelInsensitiveRequest.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ModelInsensitiveRequest.java
index 593f0807bb..3891183fed 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ModelInsensitiveRequest.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ModelInsensitiveRequest.java
@@ -22,7 +22,7 @@ import java.lang.reflect.Field;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.common.util.Unsafe;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ProjectInsensitiveRequest.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ProjectInsensitiveRequest.java
index ba98d1462c..335e3450d7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ProjectInsensitiveRequest.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/ProjectInsensitiveRequest.java
@@ -22,7 +22,7 @@ import java.lang.reflect.Field;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.common.util.Unsafe;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/UserInsensitiveRequest.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/UserInsensitiveRequest.java
index e7ed83f735..53f1443e58 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/UserInsensitiveRequest.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/insensitive/UserInsensitiveRequest.java
@@ -22,7 +22,7 @@ import java.lang.reflect.Field;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.metadata.user.ManagedUser;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 6d642c4743..40d25f80b0 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -34,7 +34,7 @@ import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.calcite.sql.util.SqlVisitor;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.measure.MeasureTypeFactory;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
index d8fb2b2964..b222252d63 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinedFlatTable.java
@@ -29,7 +29,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.calcite.avatica.util.Quoting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 22b5fa3485..5176517b8f 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
import java.util.Objects;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelJoinRelationTypeEnum.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelJoinRelationTypeEnum.java
index 0c3ac2925d..0aca46255e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelJoinRelationTypeEnum.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelJoinRelationTypeEnum.java
@@ -17,7 +17,7 @@
*/
package org.apache.kylin.metadata.model;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
index b519717da6..0e38ddb9b2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
@@ -27,7 +27,7 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.code.ErrorCodeServer;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
index 1556f6be66..c3ecda0c2b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
@@ -29,7 +29,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 13022d135d..b6cd7ebd2a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -29,7 +29,7 @@ import java.util.Set;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
import lombok.Getter;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/alias/ExpressionComparator.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/alias/ExpressionComparator.java
index 1c4d692ebc..6b98dd45e5 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/alias/ExpressionComparator.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/alias/ExpressionComparator.java
@@ -31,7 +31,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlOperator;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/AffectedModelContext.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/AffectedModelContext.java
index fb4ad339a8..3c37bccf98 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/AffectedModelContext.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/AffectedModelContext.java
@@ -24,7 +24,7 @@ import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
index ccda8a9527..57cf5e5bfe 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
@@ -25,7 +25,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexPlan;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
index b09da6b5fa..16c6e85ba6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -39,7 +39,7 @@ import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.calcite.sql.util.SqlVisitor;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ExpModifier;
import org.apache.kylin.common.util.Pair;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index 250c704991..06ac3dadd6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -50,7 +50,7 @@ import java.util.stream.Collectors;
import javax.sql.DataSource;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.jdbc.ScriptRunner;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 2620121e73..cb8b5ff0a3 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -27,7 +27,7 @@ import java.util.Objects;
import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/RawRecItemTable.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/RawRecItemTable.java
index 9e799d3bba..01113d71af 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/RawRecItemTable.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/candidate/RawRecItemTable.java
@@ -27,7 +27,7 @@ import java.sql.SQLException;
import org.apache.kylin.metadata.recommendation.entity.RecItemV2;
import org.apache.commons.lang.SerializationException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.type.JdbcType;
import org.apache.ibatis.type.TypeHandler;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/CCRecItemV2.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/CCRecItemV2.java
index 4d474e1262..9525cd6287 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/CCRecItemV2.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/entity/CCRecItemV2.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.recommendation.entity;
import java.io.Serializable;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/ref/OptRecV2.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/ref/OptRecV2.java
index 44da2716cb..404e23abb0 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/ref/OptRecV2.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/recommendation/ref/OptRecV2.java
@@ -27,7 +27,7 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
index 882a085f6d..014272635f 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
@@ -35,7 +35,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.annotation.Clarification;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfig.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfig.java
index e9a2ec0b48..fb81c6ecb4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfig.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfig.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.metadata.MetadataConstants;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
index 25289385f0..e4cf56e15a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
@@ -23,7 +23,7 @@ import static org.apache.kylin.common.exception.ServerErrorCode.PERMISSION_DENIE
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
index 4d34c5c71c..d9a66000e1 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
@@ -26,7 +26,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EpochUpdateLockManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EpochUpdateLockManagerTest.java
index 4683b6f556..4e235d8166 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EpochUpdateLockManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EpochUpdateLockManagerTest.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.junit.annotation.JdbcMetadataInfo;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.junit.Assert;
diff --git a/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
index b97461f33b..d191fc7e44 100644
--- a/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
@@ -35,7 +35,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.metrics.gauges.QueryRatioGauge;
diff --git a/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
index 087555f533..07d3b769e1 100644
--- a/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
@@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.metrics.reporter.InfluxdbReporter;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
index 83f152d7e8..9e6bcea3c7 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
@@ -53,7 +53,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index f5214019f4..ea3e921baa 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -33,7 +33,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.response.DataResult;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
index 5d8e2b4012..7ca289d990 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
@@ -28,7 +28,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.Pair;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenStreamingJobController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenStreamingJobController.java
index 800c2a4ae9..855c9cb46e 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenStreamingJobController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenStreamingJobController.java
@@ -23,7 +23,7 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java
index ee1de0d848..8ad2940b61 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java
@@ -38,7 +38,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.model.Segments;
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SampleControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SampleControllerTest.java
index 72bfbc9888..97e2b89914 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SampleControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SampleControllerTest.java
@@ -22,7 +22,7 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.nio.charset.StandardCharsets;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.job.dao.ExecutablePO;
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/StreamingJobControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/StreamingJobControllerTest.java
index b8e05d3528..84ed40219a 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/StreamingJobControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/StreamingJobControllerTest.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.SystemPropertiesCache;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
index c76f59fc2b..0b494a9c30 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
@@ -22,7 +22,7 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.nio.charset.StandardCharsets;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenStreamingJobControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenStreamingJobControllerTest.java
index 144bc1d5d1..d0c2b357cf 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenStreamingJobControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenStreamingJobControllerTest.java
@@ -23,7 +23,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PAR
import java.util.Collections;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.SystemPropertiesCache;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
index 4a0ea0f381..c974b4caeb 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
@@ -24,7 +24,7 @@ import java.util.Map;
import java.util.Optional;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index 2a97510c86..dc35003be5 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -50,7 +50,7 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.cluster.ClusterManagerFactory;
import org.apache.kylin.cluster.IClusterManager;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
index 6d4e17b81b..c42c01e6bc 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
@@ -39,7 +39,7 @@ import java.util.StringJoiner;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.JobErrorCode;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
index 431758773d..176c6281a2 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
@@ -24,7 +24,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/TableSamplingService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/TableSamplingService.java
index 698a96480f..6eddffa18f 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/TableSamplingService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/TableSamplingService.java
@@ -23,7 +23,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.kylin.engine.spark.job.NTableSamplingJob;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index c51fc1c35e..1bb130d731 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -60,7 +60,7 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
index 84fdfd3e63..6665a824e8 100644
--- a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
+++ b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/adaptor/DefaultAdaptor.java
@@ -30,7 +30,7 @@ import java.util.Map;
import javax.sql.rowset.CachedRowSet;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Joiner;
diff --git a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/JdbcConnector.java b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/JdbcConnector.java
index 32693eaf5b..86a849384c 100644
--- a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/JdbcConnector.java
+++ b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/JdbcConnector.java
@@ -27,7 +27,7 @@ import java.util.regex.Pattern;
import javax.sql.rowset.CachedRowSet;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.sdk.datasource.adaptor.AbstractJdbcAdaptor;
import org.apache.kylin.sdk.datasource.framework.conv.ConvMaster;
diff --git a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ParamNodeParser.java b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ParamNodeParser.java
index be54769363..656fffd390 100644
--- a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ParamNodeParser.java
+++ b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ParamNodeParser.java
@@ -20,7 +20,7 @@ package org.apache.kylin.sdk.datasource.framework.conv;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlConverter.java b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlConverter.java
index 6981a4d531..0e7a5f1e0e 100644
--- a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlConverter.java
+++ b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/SqlConverter.java
@@ -25,7 +25,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.dialect.CalciteSqlDialect;
import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java
index e8f2341770..a4481e30c9 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java
@@ -28,7 +28,7 @@ import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index 6b7538697c..efa49e11b1 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -61,7 +61,7 @@ import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/job-service/src/main/java/org/apache/kylin/rest/config/initialize/JobSyncListener.java b/src/job-service/src/main/java/org/apache/kylin/rest/config/initialize/JobSyncListener.java
index 1f892e7d3c..7b5f77fb9f 100644
--- a/src/job-service/src/main/java/org/apache/kylin/rest/config/initialize/JobSyncListener.java
+++ b/src/job-service/src/main/java/org/apache/kylin/rest/config/initialize/JobSyncListener.java
@@ -33,7 +33,7 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
diff --git a/src/job-service/src/test/java/org/apache/kylin/rest/config/initialize/JobSchedulerTest.java b/src/job-service/src/test/java/org/apache/kylin/rest/config/initialize/JobSchedulerTest.java
index 98a2dc6b69..0d75fb5ee8 100644
--- a/src/job-service/src/test/java/org/apache/kylin/rest/config/initialize/JobSchedulerTest.java
+++ b/src/job-service/src/test/java/org/apache/kylin/rest/config/initialize/JobSchedulerTest.java
@@ -30,7 +30,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/NKapQueryTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/NKapQueryTest.java
index b9447a22b4..c6a1912c89 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/NKapQueryTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/NKapQueryTest.java
@@ -23,7 +23,7 @@ import java.sql.SQLException;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/rest/controller/NProjectControllerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/rest/controller/NProjectControllerTest.java
index 943433c64b..8c078fd32c 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/rest/controller/NProjectControllerTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/rest/controller/NProjectControllerTest.java
@@ -23,7 +23,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import java.util.Locale;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.project.NProjectManager;
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
index 8017577c07..b520231844 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
@@ -34,7 +34,7 @@ import java.util.Locale;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.model.NDataModel;
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NProjectController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NProjectController.java
index 74b85153c3..a47868f81c 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NProjectController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NProjectController.java
@@ -39,7 +39,7 @@ import java.util.Set;
import javax.validation.Valid;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 9466eb6374..1dfd998028 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -36,8 +36,8 @@ import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserController.java
index 60b4e0797a..fa669bfc67 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserController.java
@@ -46,7 +46,7 @@ import java.util.stream.Collectors;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserGroupController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserGroupController.java
index ba30cd6c60..192812b1f6 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserGroupController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NUserGroupController.java
@@ -29,7 +29,7 @@ import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.rest.constant.Constant;
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NAccessControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NAccessControllerV2.java
index e4465dd69f..6dd3e73e45 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NAccessControllerV2.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NAccessControllerV2.java
@@ -29,7 +29,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.AclEntity;
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
index ad10f6f199..1f0e6aa271 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserControllerTest.java
index 49359f2f18..4103ea9df2 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserControllerTest.java
@@ -36,7 +36,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/model/FuzzyKeySearcher.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/model/FuzzyKeySearcher.java
index 5870223613..084a803613 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/model/FuzzyKeySearcher.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/model/FuzzyKeySearcher.java
@@ -21,7 +21,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java
index 3ec10cc795..90d2148403 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java
@@ -29,7 +29,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
index b46314b2e1..4f3781c404 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
@@ -28,7 +28,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NOT_E
import java.util.Arrays;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.metadata.acl.AclTCRManager;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java
index d4ebbbe964..2d68ea7de1 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.service;
import java.util.List;
import java.util.Locale;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
index 62d597daf5..1b9e681ef6 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
@@ -42,7 +42,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.SegmentOnlineMode;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index 725d3994f6..b7a21eb955 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -41,7 +41,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.dialect.HiveSqlDialect;
import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.CommonErrorCode;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index d83c15c751..ba966e6d28 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -103,8 +103,8 @@ import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.calcite.util.Util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
index 631de90ba6..0b38bbebfc 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/util/ModelUtils.java
@@ -26,7 +26,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.DateFormat;
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
index 10cfed3766..928c903239 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
@@ -29,7 +29,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.metadata.cube.model.IndexEntity;
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceTest.java
index a75c4a84e6..8cbb4be88e 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceTest.java
@@ -26,7 +26,7 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.scheduler.EventBusFactory;
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 3e585b7392..9c8293e8b9 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -84,9 +84,9 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index 4f47d6cc3c..5864a7e5de 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -46,7 +46,7 @@ import java.util.stream.Collectors;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index 21908533a2..d4f70edd52 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -59,7 +59,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinTimeoutException;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 7aa18f093b..ff3ba69038 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -51,7 +51,7 @@ import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.CollectionUtil;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/security/AccessDeniedException.java b/src/query-common/src/main/java/org/apache/kylin/query/security/AccessDeniedException.java
index 3ab1bff212..e45418c05c 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/security/AccessDeniedException.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/security/AccessDeniedException.java
@@ -20,7 +20,7 @@ package org.apache.kylin.query.security;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public class AccessDeniedException extends RuntimeException {
public AccessDeniedException(String s) {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java b/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java
index bb5525a3a7..f0695d63e2 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java
@@ -38,7 +38,7 @@ import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.util.SqlBasicVisitor;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.text.StrBuilder;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/AsyncQueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/AsyncQueryUtil.java
index 09e05a4e53..e05d0e003e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/AsyncQueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/AsyncQueryUtil.java
@@ -26,7 +26,7 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/ModelViewSqlNodeComparator.java b/src/query-common/src/main/java/org/apache/kylin/query/util/ModelViewSqlNodeComparator.java
index b29c2ea48e..6791a6e3d7 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/ModelViewSqlNodeComparator.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/ModelViewSqlNodeComparator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.query.util;
import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.alias.ExpressionComparator;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 4707d9406a..ddbca4afbe 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -35,7 +35,7 @@ import java.util.stream.Collectors;
import javax.ws.rs.BadRequestException;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
index 8cc71a523c..f5940b6dfb 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
@@ -41,7 +41,7 @@ import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.dialect.CalciteSqlDialect;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.Pair;
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java
index 8db117d415..8f84c63ddf 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NAsyncQueryController.java
@@ -36,7 +36,7 @@ import java.util.concurrent.atomic.AtomicReference;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index 35e2f9f86f..ea088d38a8 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -44,7 +44,7 @@ import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.ForceToTieredStorage;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
diff --git a/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkMetricsControllerTest.java b/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkMetricsControllerTest.java
index dc0833d0b6..17e71a374a 100644
--- a/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkMetricsControllerTest.java
+++ b/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkMetricsControllerTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.rest.controller;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.rest.service.MonitorService;
import org.junit.Assert;
import org.junit.Before;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java b/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java
index 7b93919c66..f9343ecf58 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.slf4j.Logger;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/ModelQueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/ModelQueryService.java
index ab70cdc64f..09b3c25d26 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/ModelQueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/ModelQueryService.java
@@ -30,7 +30,7 @@ import java.util.Set;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java
index d1c3e60617..d90adc90ee 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/MonitorService.java
@@ -27,7 +27,7 @@ import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.cluster.ClusterManagerFactory;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
index 7479631821..53520b65e4 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
@@ -40,7 +40,7 @@ import java.util.stream.Collectors;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bd6c676374..846e7b6260 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -55,7 +55,7 @@ import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.pretty.SqlPrettyWriter;
import org.apache.calcite.sql.validate.SqlValidatorException;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.ForceToTieredStorage;
import org.apache.kylin.common.KapConfig;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryCacheSignatureUtil.java b/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryCacheSignatureUtil.java
index 058867993b..81b68e074c 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryCacheSignatureUtil.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryCacheSignatureUtil.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java b/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java
index a9aba9cad5..0c922b603e 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java
@@ -18,7 +18,7 @@
package org.apache.kylin.rest.util;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.Pair;
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java b/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java
index 01fdf5e3ab..b36ccf4e92 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java
@@ -28,7 +28,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryJob.java b/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryJob.java
index c8c5aebe48..236b11a022 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryJob.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryJob.java
@@ -24,7 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/view/ModelViewTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/view/ModelViewTest.java
index 920bfebe8c..73af7b7de7 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/view/ModelViewTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/view/ModelViewTest.java
@@ -28,7 +28,7 @@ import java.util.LinkedHashMap;
import org.apache.calcite.schema.impl.ViewTable;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.cube.model.IndexPlan;
diff --git a/src/query/src/test/java/org/apache/kylin/query/schema/KylinSqlValidatorTest.java b/src/query/src/test/java/org/apache/kylin/query/schema/KylinSqlValidatorTest.java
index 93c24774b4..80e90d0f0a 100644
--- a/src/query/src/test/java/org/apache/kylin/query/schema/KylinSqlValidatorTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/schema/KylinSqlValidatorTest.java
@@ -27,7 +27,7 @@ import java.nio.file.Files;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
diff --git a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/ClickHouseStorage.java b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/ClickHouseStorage.java
index 9e5c578127..1567bdec0e 100644
--- a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/ClickHouseStorage.java
+++ b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/ClickHouseStorage.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.ClickHouseConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.SecondStorageStepFactory;
diff --git a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java
index 9822425ff8..8ac02bea48 100644
--- a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java
+++ b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouse.java
@@ -18,6 +18,13 @@
package io.kyligence.kap.clickhouse.job;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kylin.common.KylinConfig;
+
import java.io.Closeable;
import java.sql.Connection;
import java.sql.Date;
@@ -38,14 +45,6 @@ import java.util.Objects;
import java.util.Properties;
import java.util.function.Function;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.kylin.common.KylinConfig;
-
-import lombok.Getter;
-import lombok.val;
-import lombok.extern.slf4j.Slf4j;
-
@Slf4j
@Getter
public class ClickHouse implements Closeable {
diff --git a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouseLoad.java b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouseLoad.java
index ac3f1e9a97..150d989209 100644
--- a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouseLoad.java
+++ b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickHouseLoad.java
@@ -47,7 +47,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickhouseLoadFileLoad.java b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickhouseLoadFileLoad.java
index 431f583b40..4145a9108f 100644
--- a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickhouseLoadFileLoad.java
+++ b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/ClickhouseLoadFileLoad.java
@@ -21,7 +21,7 @@ package io.kyligence.kap.clickhouse.job;
import java.sql.SQLException;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import io.kyligence.kap.clickhouse.ddl.ClickHouseCreateTable;
diff --git a/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/HadoopMockUtil.java b/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/HadoopMockUtil.java
index fd9006406a..8d91f0e857 100644
--- a/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/HadoopMockUtil.java
+++ b/src/second-storage/clickhouse/src/test/java/io/kyligence/kap/clickhouse/job/HadoopMockUtil.java
@@ -21,7 +21,7 @@ package io.kyligence.kap.clickhouse.job;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.kylin.common.util.HadoopUtil;
import org.mockito.Mockito;
diff --git a/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/OpenSecondStorageEndpoint.java b/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/OpenSecondStorageEndpoint.java
index c6a89603d8..ec06ef3cc4 100644
--- a/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/OpenSecondStorageEndpoint.java
+++ b/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/OpenSecondStorageEndpoint.java
@@ -27,7 +27,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Objects;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/SecondStorageEndpoint.java b/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/SecondStorageEndpoint.java
index 1d1a9287c9..b67e565d99 100644
--- a/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/SecondStorageEndpoint.java
+++ b/src/second-storage/core-ui/src/main/java/io/kyligence/kap/secondstorage/management/SecondStorageEndpoint.java
@@ -34,7 +34,7 @@ import java.util.Map;
import java.util.Objects;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java b/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java
index 087673da05..997b981960 100644
--- a/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java
+++ b/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java
@@ -18,7 +18,7 @@
package io.kyligence.kap.secondstorage.ddl;
import io.kyligence.kap.secondstorage.ddl.visitor.DefaultSQLRender;
import io.kyligence.kap.secondstorage.ddl.visitor.Renderable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public abstract class DDL<T extends DDL<T>> implements Renderable {
diff --git a/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java b/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java
index f8c007dd34..dbb7cc119c 100644
--- a/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java
+++ b/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java
@@ -19,7 +19,7 @@ package io.kyligence.kap.secondstorage.ddl.exp;
import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
import io.kyligence.kap.secondstorage.ddl.visitor.Renderable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public class TableIdentifier implements Renderable {
public static final char TABLE_ESCAPE = '`';
diff --git a/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java b/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
index 25c5b7ba90..ce2bb65bd4 100644
--- a/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
+++ b/src/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
@@ -24,7 +24,7 @@ import java.util.function.Supplier;
import javax.annotation.concurrent.NotThreadSafe;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
diff --git a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
index 2af787aaf7..eb209b2a56 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
@@ -42,7 +42,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.ErrorCode;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java b/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java
index cfa593dca9..29b286fce4 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java
@@ -24,7 +24,7 @@ import java.util.List;
import javax.annotation.Nullable;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.discovery.KylinServiceDiscoveryCache;
import org.apache.kylin.rest.discovery.KylinServiceDiscoveryClient;
diff --git a/src/server/src/main/java/org/apache/kylin/rest/config/CorsConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/CorsConfig.java
index eb24c3136a..8c56cf07c6 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/config/CorsConfig.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/config/CorsConfig.java
@@ -17,7 +17,7 @@
*/
package org.apache.kylin.rest.config;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
diff --git a/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryCache.java b/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryCache.java
index 1fa3b8857d..e49696c18b 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryCache.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryCache.java
@@ -35,8 +35,8 @@ import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.x.discovery.ServiceCache;
diff --git a/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryClient.java b/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryClient.java
index aa7f50b9de..f9a32e74f9 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryClient.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/discovery/KylinServiceDiscoveryClient.java
@@ -27,7 +27,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.rest.response.ServerInfoResponse;
import org.springframework.cloud.client.serviceregistry.Registration;
import org.springframework.cloud.zookeeper.ConditionalOnZookeeperEnabled;
diff --git a/src/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java b/src/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
index 3a21dfc1eb..4c34d2d259 100644
--- a/src/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
+++ b/src/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
@@ -19,7 +19,7 @@
package org.apache.kylin.source.hive;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.junit.Ignore;
import org.junit.Test;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index 7239720986..db9c31c9b0 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/DefaultSparkBuildJobHandler.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/DefaultSparkBuildJobHandler.java
index 4aff2c7b55..100ce5a0be 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/DefaultSparkBuildJobHandler.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/DefaultSparkBuildJobHandler.java
@@ -30,7 +30,7 @@ import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java
index 84db20987f..0f1b68cd2d 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/ExecutableAddCuboidHandler.java
@@ -20,7 +20,7 @@ package org.apache.kylin.engine.spark.job;
import com.google.common.base.Preconditions;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.merger.AfterBuildResourceMerger;
import org.apache.kylin.job.execution.DefaultExecutableOnModel;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index 20a502ff9a..4370252206 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -237,7 +237,7 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
if (StringUtils.isEmpty(kylinJobJar) && !config.isUTEnv()) {
throw new RuntimeException("Missing kylin job jar");
}
- if (!config.isUTEnv()) {
+ if (!config.isDevOrUT()) {
sparkJobHandler.checkApplicationJar(config);
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/SparkCleanupTransactionalTableStep.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/SparkCleanupTransactionalTableStep.java
index 10974b82eb..20d0986e7f 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/SparkCleanupTransactionalTableStep.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/SparkCleanupTransactionalTableStep.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.job;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvSource.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvSource.java
index c0be9c5f69..ec0f7229d5 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvSource.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvSource.java
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvTableReader.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvTableReader.java
index 9a11d1e29b..959562ca1f 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvTableReader.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/mockup/CsvTableReader.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.IReadableTable;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkDataSource.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkDataSource.java
index 6a501bc71e..61882f3dcb 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkDataSource.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkDataSource.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.source;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.SegmentRange;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
index c037c37bfe..cf9508dec6 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
@@ -29,7 +29,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/analyzer/TableAnalyzerJob.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/analyzer/TableAnalyzerJob.java
index 25ef403501..a17bcf5997 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/analyzer/TableAnalyzerJob.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/analyzer/TableAnalyzerJob.java
@@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import lombok.val;
import lombok.var;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/utils/DateTimeCheckUtils.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/utils/DateTimeCheckUtils.java
index 46ea5acf79..468379c723 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/utils/DateTimeCheckUtils.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/stats/utils/DateTimeCheckUtils.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.stats.utils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
public class DateTimeCheckUtils {
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/HiveTransactionTableHelper.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/HiveTransactionTableHelper.java
index 5a65f573d5..6810e58828 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/HiveTransactionTableHelper.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/HiveTransactionTableHelper.java
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Objects;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/SparkJobFactoryUtils.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/SparkJobFactoryUtils.java
index 8f9c7223c3..9a666dee30 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/SparkJobFactoryUtils.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/SparkJobFactoryUtils.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.utils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.engine.spark.job.NSparkCubingJob;
import org.apache.kylin.engine.spark.job.NSparkCubingStep;
import org.apache.kylin.engine.spark.job.NSparkMergingJob;
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
index 6bbf5217c7..e3c930c883 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
@@ -22,7 +22,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.kylin.common.KylinConfig;
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java
index 08cc585aba..9bb9e12999 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentMergeJob.java
@@ -20,7 +20,7 @@ package org.apache.kylin.engine.spark.job;
import com.google.common.base.Throwables;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.job.exec.MergeExec;
import org.apache.kylin.engine.spark.job.stage.BuildParam;
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SnapshotBuildJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SnapshotBuildJob.java
index b1f4e13477..2202493c0c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SnapshotBuildJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SnapshotBuildJob.java
@@ -22,7 +22,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import lombok.val;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcSource.java b/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcSource.java
index 4c49a82a00..55de85c6a2 100644
--- a/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcSource.java
+++ b/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcSource.java
@@ -21,7 +21,7 @@ import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_JDBC_SOU
import java.io.IOException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
index 045850a7ef..c22178e9d7 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
@@ -28,7 +28,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.calcite.avatica.util.Quoting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.metadata.cube.model.IndexPlan;
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java
index db09039892..3034b2e8bc 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.nio.charset.Charset;
import java.util.TreeSet;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
diff --git a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
index 2cdb832339..fe44575072 100644
--- a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.common.asyncprofiler;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.junit.After;
import org.junit.Assert;
diff --git a/src/streaming-service/src/main/java/org/apache/kylin/rest/service/StreamingJobService.java b/src/streaming-service/src/main/java/org/apache/kylin/rest/service/StreamingJobService.java
index 3658119bcd..01e80bbe0f 100644
--- a/src/streaming-service/src/main/java/org/apache/kylin/rest/service/StreamingJobService.java
+++ b/src/streaming-service/src/main/java/org/apache/kylin/rest/service/StreamingJobService.java
@@ -38,7 +38,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ObjectUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
diff --git a/src/streaming/src/main/java/org/apache/kylin/kafka/util/KafkaUtils.java b/src/streaming/src/main/java/org/apache/kylin/kafka/util/KafkaUtils.java
index 26bb14d07d..21eb10fe63 100644
--- a/src/streaming/src/main/java/org/apache/kylin/kafka/util/KafkaUtils.java
+++ b/src/streaming/src/main/java/org/apache/kylin/kafka/util/KafkaUtils.java
@@ -24,7 +24,7 @@ import java.nio.ByteBuffer;
import java.util.Properties;
import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.KafkaConsumer;
diff --git a/src/streaming/src/main/java/org/apache/kylin/streaming/jobs/AbstractSparkJobLauncher.java b/src/streaming/src/main/java/org/apache/kylin/streaming/jobs/AbstractSparkJobLauncher.java
index 5ebbf1fc5e..651d840caf 100644
--- a/src/streaming/src/main/java/org/apache/kylin/streaming/jobs/AbstractSparkJobLauncher.java
+++ b/src/streaming/src/main/java/org/apache/kylin/streaming/jobs/AbstractSparkJobLauncher.java
@@ -25,7 +25,7 @@ import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.job.execution.JobTypeEnum;
diff --git a/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java b/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
index 0a8f8b1a20..777a4830f5 100644
--- a/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
+++ b/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
@@ -20,7 +20,7 @@ package org.apache.kylin.streaming.manager;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.job.constant.JobStatusEnum;
diff --git a/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java b/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
index 21d34f0fba..55f4529422 100644
--- a/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
+++ b/src/streaming/src/main/java/org/apache/kylin/streaming/rest/RestSupport.java
@@ -23,7 +23,7 @@ import java.io.Closeable;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java b/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java
index 8548fea87e..bcbc837fc5 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java
@@ -58,7 +58,7 @@ import javax.xml.bind.DatatypeConverter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinTimeoutException;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/AuditLogTool.java b/src/tool/src/main/java/org/apache/kylin/tool/AuditLogTool.java
index 7c5a34915c..66c2d6ef29 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/AuditLogTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/AuditLogTool.java
@@ -45,7 +45,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.dbcp2.BasicDataSourceFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.ExecutableApplication;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/ClickhouseDiagTool.java b/src/tool/src/main/java/org/apache/kylin/tool/ClickhouseDiagTool.java
index a03e2dfa00..5231b36c4b 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/ClickhouseDiagTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/ClickhouseDiagTool.java
@@ -29,8 +29,8 @@ import java.util.zip.GZIPInputStream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Pair;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/ConfTool.java b/src/tool/src/main/java/org/apache/kylin/tool/ConfTool.java
index 1c01adac45..852cc99a38 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/ConfTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/ConfTool.java
@@ -22,7 +22,7 @@ import java.nio.file.Files;
import java.util.Set;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.tool.util.ToolUtil;
import org.slf4j.Logger;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/DumpHadoopSystemProps.java b/src/tool/src/main/java/org/apache/kylin/tool/DumpHadoopSystemProps.java
index 862d5df713..4170bd766e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/DumpHadoopSystemProps.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/DumpHadoopSystemProps.java
@@ -29,7 +29,7 @@ import java.util.Arrays;
import java.util.Map;
import java.util.TreeMap;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.Unsafe;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java b/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
index 2f47669fa5..17a43b59e3 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/JobDiagInfoTool.java
@@ -33,7 +33,7 @@ import java.util.stream.Collectors;
import com.google.common.collect.Sets;
import org.apache.commons.cli.Option;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinRuntimeException;
import org.apache.kylin.common.util.OptionsHelper;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java b/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
index 612e6dc7ce..3ef20b0bbf 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/KylinLogTool.java
@@ -46,7 +46,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/KapGuardian.java b/src/tool/src/main/java/org/apache/kylin/tool/daemon/KapGuardian.java
index fab7868269..a876d2a7fe 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/daemon/KapGuardian.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/daemon/KapGuardian.java
@@ -28,7 +28,7 @@ import java.util.stream.Collectors;
import org.apache.calcite.avatica.util.Unsafe;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.ExecutorServiceUtil;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/obf/KylinConfObfuscator.java b/src/tool/src/main/java/org/apache/kylin/tool/obf/KylinConfObfuscator.java
index ff168842a8..9d9651a052 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/obf/KylinConfObfuscator.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/obf/KylinConfObfuscator.java
@@ -20,7 +20,7 @@ package org.apache.kylin.tool.obf;
import java.util.Map;
import java.util.Properties;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.StorageURL;
import org.apache.kylin.tool.constant.SensitiveConfigKeysConstant;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/CheckProjectModeCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/CheckProjectModeCLI.java
index a68139a5ef..58ad2d14c7 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/CheckProjectModeCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/CheckProjectModeCLI.java
@@ -30,7 +30,7 @@ import java.util.Locale;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionsHelper;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/DeleteFavoriteQueryCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/DeleteFavoriteQueryCLI.java
index bba14a41c2..54200f7b86 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/DeleteFavoriteQueryCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/DeleteFavoriteQueryCLI.java
@@ -27,7 +27,7 @@ import java.util.Map;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionsHelper;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameEntity.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameEntity.java
index 11f3f22ce6..7ab86f3217 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameEntity.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameEntity.java
@@ -23,7 +23,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Locale;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameUserResourceTool.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameUserResourceTool.java
index 015fd590c8..657153367b 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameUserResourceTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/RenameUserResourceTool.java
@@ -42,7 +42,7 @@ import java.util.stream.Collectors;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateModelCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateModelCLI.java
index 972967dc59..2569a75a7e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateModelCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateModelCLI.java
@@ -30,7 +30,7 @@ import java.util.stream.Collectors;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionsHelper;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateProjectCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateProjectCLI.java
index 42f560c163..ef8d821c0d 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateProjectCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateProjectCLI.java
@@ -26,7 +26,7 @@ import java.util.Map;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionsHelper;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateSessionTableCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateSessionTableCLI.java
index 12db92db95..586ffe5ac5 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateSessionTableCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateSessionTableCLI.java
@@ -29,7 +29,7 @@ import javax.sql.DataSource;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionsHelper;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserGroupCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserGroupCLI.java
index 7b406da2cb..5859df7b10 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserGroupCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserGroupCLI.java
@@ -29,7 +29,7 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.JsonUtil;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/HadoopConfExtractor.java b/src/tool/src/main/java/org/apache/kylin/tool/util/HadoopConfExtractor.java
index bc92f7f14c..9488669c27 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/util/HadoopConfExtractor.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/HadoopConfExtractor.java
@@ -21,7 +21,7 @@ package org.apache.kylin.tool.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/ServerInfoUtil.java b/src/tool/src/main/java/org/apache/kylin/tool/util/ServerInfoUtil.java
index 6a250bfddc..2047330b68 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/util/ServerInfoUtil.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/ServerInfoUtil.java
@@ -21,7 +21,7 @@ import java.io.File;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/StorageCleanerTest.java b/src/tool/src/test/java/org/apache/kylin/tool/StorageCleanerTest.java
index 9315dcdb7c..5dca0c1680 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/StorageCleanerTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/StorageCleanerTest.java
@@ -27,7 +27,7 @@ import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/garbage/SnapshotCleanerTest.java b/src/tool/src/test/java/org/apache/kylin/tool/garbage/SnapshotCleanerTest.java
index 634345c66b..90fd332091 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/garbage/SnapshotCleanerTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/garbage/SnapshotCleanerTest.java
@@ -21,7 +21,7 @@ package org.apache.kylin.tool.garbage;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/general/CryptToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/general/CryptToolTest.java
index ab7dd8938a..714aa9b9c4 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/general/CryptToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/general/CryptToolTest.java
@@ -23,7 +23,7 @@ import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/upgrade/RenameUserResourceToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/upgrade/RenameUserResourceToolTest.java
index 3219873a00..bfa47ef0aa 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/upgrade/RenameUserResourceToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/upgrade/RenameUserResourceToolTest.java
@@ -24,7 +24,7 @@ import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Locale;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
[kylin] 05/38: KYLIN-5521 fix JoinsGraph
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2502c156320934204b333d3c746f820ad1963ecf
Author: Jiale He <ji...@gmail.com>
AuthorDate: Wed Feb 1 17:41:37 2023 +0800
KYLIN-5521 fix JoinsGraph
---
.../apache/kylin/metadata/model/JoinsGraph.java | 610 ---------------------
.../kylin/metadata/model/graph/JoinsGraph.java | 2 +-
.../kylin/metadata/model/JoinsGraphTest.java | 4 +-
3 files changed, 4 insertions(+), 612 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
deleted file mode 100644
index acd810fbec..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsGraph.java
+++ /dev/null
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.metadata.model;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import lombok.Getter;
-import lombok.NonNull;
-import lombok.Setter;
-
-public class JoinsGraph implements Serializable {
-
- public class Edge implements Serializable {
-
- @Getter
- private final JoinDesc join;
- private final ColumnDesc[] leftCols;
- private final ColumnDesc[] rightCols;
- private final NonEquiJoinCondition nonEquiJoinCondition;
-
- private Edge(JoinDesc join) {
- this.join = join;
-
- leftCols = new ColumnDesc[join.getForeignKeyColumns().length];
- int i = 0;
- for (TblColRef colRef : join.getForeignKeyColumns()) {
- leftCols[i++] = colRef.getColumnDesc();
- }
-
- rightCols = new ColumnDesc[join.getPrimaryKeyColumns().length];
- i = 0;
- for (TblColRef colRef : join.getPrimaryKeyColumns()) {
- rightCols[i++] = colRef.getColumnDesc();
- }
-
- nonEquiJoinCondition = join.getNonEquiJoinCondition();
- }
-
- public boolean isJoinMatched(JoinDesc other) {
- return join.equals(other);
- }
-
- public boolean isNonEquiJoin() {
- return nonEquiJoinCondition != null;
- }
-
- public boolean isLeftJoin() {
- return !join.isLeftOrInnerJoin() && join.isLeftJoin();
- }
-
- public boolean isLeftOrInnerJoin() {
- return join.isLeftOrInnerJoin();
- }
-
- public boolean isInnerJoin() {
- return !join.isLeftOrInnerJoin() && join.isInnerJoin();
- }
-
- private TableRef left() {
- return join.getFKSide();
- }
-
- private TableRef right() {
- return join.getPKSide();
- }
-
- private boolean isFkSide(TableRef tableRef) {
- return join.getFKSide().equals(tableRef);
- }
-
- private boolean isPkSide(TableRef tableRef) {
- return join.getPKSide().equals(tableRef);
- }
-
- private TableRef other(TableRef tableRef) {
- if (left().equals(tableRef)) {
- return right();
- } else if (right().equals(tableRef)) {
- return left();
- }
- throw new IllegalArgumentException("table " + tableRef + " is not on the edge " + this);
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
-
- if (this.getClass() != that.getClass())
- return false;
-
- return joinEdgeMatcher.matches(this, (Edge) that);
- }
-
- @Override
- public int hashCode() {
- if (this.isLeftJoin()) {
- return Objects.hash(isLeftJoin(), leftCols, rightCols);
- } else {
- if (Arrays.hashCode(leftCols) < Arrays.hashCode(rightCols)) {
- return Objects.hash(isLeftJoin(), leftCols, rightCols);
- } else {
- return Objects.hash(isLeftJoin(), rightCols, leftCols);
- }
- }
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("Edge: ");
- sb.append(left())
- .append(isLeftJoin() ? " LEFT JOIN "
- : isLeftOrInnerJoin() ? " LEFT OR INNER JOIN " : " INNER JOIN ")
- .append(right()).append(" ON ")
- .append(Arrays.toString(Arrays.stream(leftCols).map(ColumnDesc::getName).toArray())).append(" = ")
- .append(Arrays.toString(Arrays.stream(rightCols).map(ColumnDesc::getName).toArray()));
- return sb.toString();
- }
- }
-
- private Edge edgeOf(JoinDesc join) {
- return new Edge(join);
- }
-
- private static final IJoinEdgeMatcher DEFAULT_JOIN_EDGE_MATCHER = new DefaultJoinEdgeMatcher();
- @Setter
- private IJoinEdgeMatcher joinEdgeMatcher = DEFAULT_JOIN_EDGE_MATCHER;
-
- /**
- * compare:
- * 1. JoinType
- * 2. Columns on both sides
- */
- public interface IJoinEdgeMatcher extends Serializable {
- boolean matches(@NonNull Edge join1, @NonNull Edge join2);
- }
-
- public static class DefaultJoinEdgeMatcher implements IJoinEdgeMatcher {
- @Override
- public boolean matches(@NonNull Edge join1, @NonNull Edge join2) {
- if (join1.isLeftJoin() != join2.isLeftJoin() && !join1.isLeftOrInnerJoin() && !join2.isLeftOrInnerJoin()) {
- return false;
- }
-
- if (!Objects.equals(join1.nonEquiJoinCondition, join2.nonEquiJoinCondition)) {
- return false;
- }
-
- if (join1.isLeftJoin()) {
- return columnDescEquals(join1.leftCols, join2.leftCols)
- && columnDescEquals(join1.rightCols, join2.rightCols);
- } else {
- return (columnDescEquals(join1.leftCols, join2.leftCols)
- && columnDescEquals(join1.rightCols, join2.rightCols))
- || (columnDescEquals(join1.leftCols, join2.rightCols)
- && columnDescEquals(join1.rightCols, join2.leftCols));
- }
- }
-
- private boolean columnDescEquals(ColumnDesc[] a, ColumnDesc[] b) {
- if (a.length != b.length) {
- return false;
- }
-
- List<ColumnDesc> oneList = Lists.newArrayList(a);
- List<ColumnDesc> anotherList = Lists.newArrayList(b);
-
- for (ColumnDesc obj : oneList) {
- anotherList.removeIf(dual -> columnDescEquals(obj, dual));
- }
- return anotherList.isEmpty();
- }
-
- protected boolean columnDescEquals(ColumnDesc a, ColumnDesc b) {
- return Objects.equals(a, b);
- }
- }
-
- @Getter
- private final TableRef center;
- private final Map<String, TableRef> nodes = new HashMap<>();
- private final Set<Edge> edges = new HashSet<>();
- private final Map<TableRef, List<Edge>> edgesFromNode = new HashMap<>();
- private final Map<TableRef, List<Edge>> edgesToNode = new HashMap<>();
-
- /**
- * For model there's always a center, if there's only one tableScan it's the center.
- * Otherwise the center is not determined, it's a linked graph, hard to tell the center.
- */
- public JoinsGraph(TableRef root, List<JoinDesc> joins) {
- this.center = root;
- addNode(root);
-
- for (JoinDesc join : joins) {
- Preconditions.checkState(Arrays.stream(join.getForeignKeyColumns()).allMatch(TblColRef::isQualified));
- Preconditions.checkState(Arrays.stream(join.getPrimaryKeyColumns()).allMatch(TblColRef::isQualified));
- addAsEdge(join);
- }
-
- validate(joins);
- }
-
- private void addNode(TableRef table) {
- Preconditions.checkNotNull(table);
- String alias = table.getAlias();
- TableRef node = nodes.get(alias);
- if (node != null) {
- Preconditions.checkArgument(node.equals(table), "[%s]'s Alias \"%s\" has conflict with [%s].", table, alias,
- node);
- } else {
- nodes.put(alias, table);
- }
- }
-
- private void addAsEdge(JoinDesc join) {
- TableRef fkTable = join.getFKSide();
- TableRef pkTable = join.getPKSide();
- addNode(pkTable);
-
- Edge edge = edgeOf(join);
- edgesFromNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
- edgesFromNode.get(fkTable).add(edge);
- edgesToNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
- edgesToNode.get(pkTable).add(edge);
- if (!edge.isLeftJoin()) {
- // inner join is reversible
- edgesFromNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
- edgesFromNode.get(pkTable).add(edge);
- edgesToNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
- edgesToNode.get(fkTable).add(edge);
- }
- edges.add(edge);
- }
-
- public void setJoinToLeftOrInner(JoinDesc join) {
- if (!join.isLeftJoin()) {
- join.setLeftOrInner(true);
- return;
- }
-
- join.setLeftOrInner(true);
- TableRef fkTable = join.getFKSide();
- TableRef pkTable = join.getPKSide();
- Edge edge = edges.stream().filter(e -> e.isJoinMatched(join)).findFirst().orElse(null);
- if (edge == null) {
- return;
- }
- edgesFromNode.computeIfAbsent(pkTable, pk -> Lists.newArrayList());
- edgesFromNode.get(pkTable).add(edge);
- edgesToNode.computeIfAbsent(fkTable, fk -> Lists.newArrayList());
- edgesToNode.get(fkTable).add(edge);
- }
-
- private void validate(List<JoinDesc> joins) {
- for (JoinDesc join : joins) {
- TableRef fkTable = join.getFKSide();
- Preconditions.checkNotNull(nodes.get(fkTable.getAlias()));
- Preconditions.checkState(nodes.get(fkTable.getAlias()).equals(fkTable));
- }
- Preconditions.checkState(nodes.size() == joins.size() + 1);
- }
-
- public boolean match(JoinsGraph pattern, Map<String, String> matchAlias) {
- return match(pattern, matchAlias, false);
- }
-
- public boolean match(JoinsGraph pattern, Map<String, String> matchAlias, boolean matchPatial) {
- return match(pattern, matchAlias, matchPatial, false);
- }
-
- public boolean match(JoinsGraph pattern, Map<String, String> matchAlias, boolean matchPatial,
- boolean matchPartialNonEquiJoin) {
- if (pattern == null || pattern.center == null) {
- throw new IllegalArgumentException("pattern(model) should have a center: " + pattern);
- }
-
- List<TableRef> candidatesOfQCenter = searchCenterByIdentity(pattern.center);
- if (CollectionUtils.isEmpty(candidatesOfQCenter)) {
- return false;
- }
-
- for (TableRef queryCenter : candidatesOfQCenter) {
- // query <-> pattern
- Map<TableRef, TableRef> trialMatch = Maps.newHashMap();
- trialMatch.put(queryCenter, pattern.center);
-
- if (!checkInnerJoinNum(pattern, queryCenter, pattern.center, matchPatial)) {
- continue;
- }
-
- AtomicReference<Map<TableRef, TableRef>> finalMatchRef = new AtomicReference<>();
- innerMatch(pattern, trialMatch, matchPatial, finalMatchRef);
- if (finalMatchRef.get() != null
- && (matchPartialNonEquiJoin || checkNonEquiJoinMatches(finalMatchRef.get(), pattern))) {
- matchAlias.clear();
- matchAlias.putAll(finalMatchRef.get().entrySet().stream()
- .collect(Collectors.toMap(e -> e.getKey().getAlias(), e -> e.getValue().getAlias())));
- return true;
- }
- }
- return false;
- }
-
- public static JoinsGraph normalizeJoinGraph(JoinsGraph joinsGraph) {
- for (Edge edge : joinsGraph.edges) {
- if (!edge.isLeftJoin() || edge.isLeftOrInnerJoin()) {
- TableRef leftTable = edge.left();
- List<Edge> edgeList = joinsGraph.edgesToNode.get(leftTable);
- if (CollectionUtils.isEmpty(edgeList)) {
- continue;
- }
- for (Edge targetEdge : edgeList) {
- if (!edge.equals(targetEdge) && leftTable.equals(targetEdge.right())
- && !targetEdge.isLeftOrInnerJoin()) {
- joinsGraph.setJoinToLeftOrInner(targetEdge.join);
- normalizeJoinGraph(joinsGraph);
- }
- }
- }
- }
- return joinsGraph;
- }
-
- public List<TableRef> getAllTblRefNodes() {
- return Lists.newArrayList(nodes.values());
- }
-
- /**
- * check if any non-equi join is missed in the pattern
- * if so, we cannot match the current graph with the the pattern graph.
- * set `kylin.query.match-partial-non-equi-join-model` to skip this checking
- * @param matches
- * @return
- */
- private boolean checkNonEquiJoinMatches(Map<TableRef, TableRef> matches, JoinsGraph pattern) {
- HashSet<TableRef> patternGraphTables = new HashSet<>(pattern.nodes.values());
-
- for (TableRef patternTable : patternGraphTables) {
- List<Edge> outgoingEdges = pattern.getEdgesByFKSide(patternTable);
- // for all outgoing non-equi join edges
- // if there is no match found for the right side table in the current graph
- // return false
- for (Edge outgoingEdge : outgoingEdges) {
- if (outgoingEdge.isNonEquiJoin()) {
- if (!matches.containsValue(patternTable) || !matches.containsValue(outgoingEdge.right())) {
- return false;
- }
- }
- }
- }
- return true;
- }
-
- private boolean isAllJoinInner(JoinsGraph joinsGraph, TableRef tableRef) {
- List<Edge> edgesFromNode = joinsGraph.edgesFromNode.get(tableRef);
- List<Edge> edgesToNode = joinsGraph.edgesToNode.get(tableRef);
-
- if (edgesFromNode == null) {
- return false;
- }
-
- if (edgesToNode == null) {
- return false;
- }
-
- if (edgesToNode.size() != edgesFromNode.size()) {
- return false;
- }
-
- for (Edge edge : edgesFromNode) {
- if (edge.join.isLeftJoin()) {
- return false;
- }
- }
-
- return true;
- }
-
- private boolean checkInnerJoinNum(JoinsGraph pattern, TableRef queryTableRef, TableRef patternTableRef,
- boolean matchPartial) {
- if (matchPartial) {
- return true;
- }
- // fully match: unmatched if extra inner join edge on either graph
- // matched if: query graph join count: model graph join count:
- // 1) inner join <= inner join
- // 2) inner join + left or inner join >= inner join
- List<Edge> innerQueryEdges = this.edgesFrom(queryTableRef).stream().filter(Edge::isInnerJoin)
- .collect(Collectors.toList());
- List<Edge> notLeftQueryEdges = this.edgesFrom(queryTableRef).stream().filter(e -> !e.isLeftJoin())
- .collect(Collectors.toList());
- List<Edge> innerPatternEdges = pattern.edgesFrom(patternTableRef).stream().filter(Edge::isInnerJoin)
- .collect(Collectors.toList());
-
- // if all joins are inner joins, compare sum of both sides
- if (isAllJoinInner(this, queryTableRef) && isAllJoinInner(pattern, patternTableRef)) {
- int cntInnerQueryEdges = innerQueryEdges.size();
- int cntNotLeftQueryEdges = notLeftQueryEdges.size();
- int cntInnerPatternEdges = innerPatternEdges.size();
- return cntInnerQueryEdges <= cntInnerPatternEdges && cntNotLeftQueryEdges >= cntInnerPatternEdges;
- }
-
- // if not all joins are inner, compare left side and right side separately
- // Calculate join count in query graph
- int cntLeftSideInnerQueryEdges = (int) innerQueryEdges.stream()
- .filter(edge -> edge.right().equals(queryTableRef)).count();
- int cntRightSideInnerQueryEdges = (int) innerQueryEdges.stream()
- .filter(edge -> edge.left().equals(queryTableRef)).count();
- int cntLeftSideNotLeftQueryEdges = (int) notLeftQueryEdges.stream()
- .filter(edge -> edge.right().equals(queryTableRef)).count();
- int cntRightSideNotLeftQueryEdges = (int) notLeftQueryEdges.stream()
- .filter(edge -> edge.left().equals(queryTableRef)).count();
- // Calculate join count in model graph
- int cntLeftSideInnerPatternEdges = (int) innerPatternEdges.stream()
- .filter(edge -> edge.right().equals(patternTableRef)).count();
- int cntRightSideInnerPatternEdges = (int) innerPatternEdges.stream()
- .filter(edge -> edge.left().equals(patternTableRef)).count();
-
- boolean isLeftEqual = cntLeftSideInnerQueryEdges <= cntLeftSideInnerPatternEdges
- && cntLeftSideNotLeftQueryEdges >= cntLeftSideInnerPatternEdges;
- boolean isRightEqual = cntRightSideInnerQueryEdges <= cntRightSideInnerPatternEdges
- && cntRightSideNotLeftQueryEdges >= cntRightSideInnerPatternEdges;
- return isLeftEqual && isRightEqual;
- }
-
- private void innerMatch(JoinsGraph pattern, Map<TableRef, TableRef> trialMatches, boolean matchPartial,
- AtomicReference<Map<TableRef, TableRef>> finalMatch) {
- if (trialMatches.size() == nodes.size()) {
- //match is found
- finalMatch.set(trialMatches);
- return;
- }
-
- Preconditions.checkState(nodes.size() > trialMatches.size());
- Optional<Pair<Edge, TableRef>> toMatch = trialMatches.keySet().stream()
- .map(t -> edgesFrom(t).stream().filter(e -> !trialMatches.containsKey(e.other(t))).findFirst()
- .map(edge -> new Pair<>(edge, edge.other(t))).orElse(null))
- .filter(Objects::nonNull).findFirst();
-
- Preconditions.checkState(toMatch.isPresent());
- Edge toMatchQueryEdge = toMatch.get().getFirst();
- TableRef toMatchQueryNode = toMatch.get().getSecond();
- TableRef matchedQueryNode = toMatchQueryEdge.other(toMatchQueryNode);
- TableRef matchedPatternNode = trialMatches.get(matchedQueryNode);
-
- List<TableRef> toMatchPatternNodeCandidates = Lists.newArrayList();
- for (Edge patternEdge : pattern.edgesFrom(matchedPatternNode)) {
- TableRef toMatchPatternNode = patternEdge.other(matchedPatternNode);
- if (!toMatchQueryNode.getTableIdentity().equals(toMatchPatternNode.getTableIdentity())
- || !toMatchQueryEdge.equals(patternEdge) || trialMatches.containsValue(toMatchPatternNode)
- || !checkInnerJoinNum(pattern, toMatchQueryNode, toMatchPatternNode, matchPartial)) {
- continue;
- }
- toMatchPatternNodeCandidates.add(toMatchPatternNode);
- }
-
- for (TableRef toMatchPatternNode : toMatchPatternNodeCandidates) {
- Map<TableRef, TableRef> newTrialMatches = Maps.newHashMap();
- newTrialMatches.putAll(trialMatches);
- newTrialMatches.put(toMatchQueryNode, toMatchPatternNode);
- innerMatch(pattern, newTrialMatches, matchPartial, finalMatch);
- if (finalMatch.get() != null) {
- //get out of recursive invoke chain straightly
- return;
- }
- }
- }
-
- public List<Edge> unmatched(JoinsGraph pattern) {
- List<Edge> unmatched = Lists.newArrayList();
- Set<Edge> all = edgesFromNode.values().stream().flatMap(List::stream).collect(Collectors.toSet());
- for (Edge edge : all) {
- List<JoinDesc> joins = getJoinsPathByPKSide(edge.right());
- JoinsGraph subGraph = new JoinsGraph(center, joins);
- if (subGraph.match(pattern, Maps.newHashMap())) {
- continue;
- }
- unmatched.add(edge);
- }
- return unmatched;
- }
-
- private List<TableRef> searchCenterByIdentity(final TableRef table) {
- // special case: several same nodes in a JoinGraph
- return nodes.values().stream().filter(node -> node.getTableIdentity().equals(table.getTableIdentity()))
- .filter(node -> {
- List<JoinDesc> path2Center = getJoinsPathByPKSide(node);
- return path2Center.stream().noneMatch(JoinDesc::isLeftJoin);
- }).collect(Collectors.toList());
- }
-
- private List<Edge> edgesFrom(TableRef thisSide) {
- return edgesFromNode.getOrDefault(thisSide, Lists.newArrayList());
- }
-
- public Map<String, String> matchAlias(JoinsGraph joinsGraph, KylinConfig kylinConfig) {
- Map<String, String> matchAlias = Maps.newHashMap();
- match(joinsGraph, matchAlias, kylinConfig.isQueryMatchPartialInnerJoinModel(),
- kylinConfig.partialMatchNonEquiJoins());
- return matchAlias;
- }
-
- public Map<String, String> matchAlias(JoinsGraph joinsGraph, boolean matchPartial) {
- Map<String, String> matchAlias = Maps.newHashMap();
- match(joinsGraph, matchAlias, matchPartial);
- return matchAlias;
- }
-
- public List<Edge> getEdgesByFKSide(TableRef table) {
- if (!edgesFromNode.containsKey(table)) {
- return Lists.newArrayList();
- }
- return edgesFromNode.get(table).stream().filter(e -> e.isFkSide(table)).collect(Collectors.toList());
- }
-
- private Edge getEdgeByPKSide(TableRef table) {
- if (!edgesToNode.containsKey(table)) {
- return null;
- }
- List<Edge> edgesByPkSide = edgesToNode.get(table).stream().filter(e -> e.isPkSide(table))
- .collect(Collectors.toList());
- if (edgesByPkSide.isEmpty()) {
- return null;
- }
- Preconditions.checkState(edgesByPkSide.size() == 1, "%s is allowed to be Join PK side once", table);
- return edgesByPkSide.get(0);
- }
-
- public JoinDesc getJoinByPKSide(TableRef table) {
- Edge edge = getEdgeByPKSide(table);
- return edge != null ? edge.join : null;
- }
-
- private List<JoinDesc> getJoinsPathByPKSide(TableRef table) {
- List<JoinDesc> pathToRoot = Lists.newArrayList();
- TableRef pkSide = table; // start from leaf
- while (pkSide != null) {
- JoinDesc subJoin = getJoinByPKSide(pkSide);
- if (subJoin != null) {
- pathToRoot.add(subJoin);
- pkSide = subJoin.getFKSide();
- } else {
- pkSide = null;
- }
- }
- return Lists.reverse(pathToRoot);
- }
-
- public JoinsGraph getSubgraphByAlias(Set<String> aliasSets) {
- TableRef subGraphRoot = this.center;
- Set<JoinDesc> subGraphJoin = Sets.newHashSet();
- for (String alias : aliasSets) {
- subGraphJoin.addAll(getJoinsPathByPKSide(nodes.get(alias)));
- }
- return new JoinsGraph(subGraphRoot, Lists.newArrayList(subGraphJoin));
- }
-
- @Override
- public String toString() {
- StringBuilder graphStrBuilder = new StringBuilder();
- graphStrBuilder.append("Root: ").append(center);
- List<Edge> nextEdges = getEdgesByFKSide(center);
- nextEdges.forEach(e -> buildGraphStr(graphStrBuilder, e, 1));
- return graphStrBuilder.toString();
- }
-
- private void buildGraphStr(StringBuilder sb, @NonNull Edge edge, int indent) {
- sb.append('\n');
- for (int i = 0; i < indent; i++) {
- sb.append(" ");
- }
- sb.append(edge);
- List<Edge> nextEdges = getEdgesByFKSide(edge.right());
- nextEdges.forEach(e -> buildGraphStr(sb, e, indent + 1));
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
index 67f8e26bfc..b6342fd476 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/graph/JoinsGraph.java
@@ -355,7 +355,7 @@ public class JoinsGraph implements Serializable {
}
for (Edge targetEdge : edgeList) {
if (!edge.equals(targetEdge) && fkSide.equals(targetEdge.pkSide())
- && !targetEdge.isLeftOrInnerJoin()) {
+ && !targetEdge.isLeftOrInnerJoin() && targetEdge.isLeftJoin()) {
setJoinToLeftOrInner(targetEdge.join);
normalize();
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
index 65ad1e25da..65a359e63f 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsGraphTest.java
@@ -29,6 +29,8 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.metadata.model.graph.DefaultJoinEdgeMatcher;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.project.NProjectManager;
import org.junit.Assert;
import org.junit.Before;
@@ -349,7 +351,7 @@ public class JoinsGraphTest extends NLocalFileMetadataTestCase {
public void testColumnDescEquals() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
NTableMetadataManager manager = NTableMetadataManager.getInstance(getTestConfig(), "default");
TableDesc tableDesc = manager.getTableDesc("DEFAULT.TEST_KYLIN_FACT");
- JoinsGraph.DefaultJoinEdgeMatcher matcher = new JoinsGraph.DefaultJoinEdgeMatcher();
+ DefaultJoinEdgeMatcher matcher = new DefaultJoinEdgeMatcher();
ColumnDesc one = new ColumnDesc();
one.setTable(tableDesc);
one.setName("one");
[kylin] 28/38: mirror: fix snyk, upgrade tomcat-embed-core from 9.0.69 to 9.0.71 fix snyk, upgrade commons-fileupload from 1.3.3 to 1.5
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 10a180422c2544dc82c2205d76096e5fc39624e0
Author: huangsheng <hu...@163.com>
AuthorDate: Thu Mar 2 10:26:34 2023 +0800
mirror: fix snyk, upgrade tomcat-embed-core from 9.0.69 to 9.0.71
fix snyk, upgrade commons-fileupload from 1.3.3 to 1.5
---
pom.xml | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 53a49fb81c..f7a6740b8f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -146,7 +146,7 @@
<commons-cli.version>1.2</commons-cli.version>
<commons-lang.version>2.6</commons-lang.version>
<commons-io.version>2.7</commons-io.version>
- <commons-upload.version>1.3.3</commons-upload.version>
+ <commons-upload.version>1.5</commons-upload.version>
<commons-math3.version>3.4.1</commons-math3.version>
<commons-collections.version>3.2.2</commons-collections.version>
<commons-codec.version>1.10</commons-codec.version>
@@ -2363,6 +2363,12 @@
<artifactId>spring-session-data-redis</artifactId>
<version>${spring-session.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tomcat.embed</groupId>
+ <artifactId>tomcat-embed-core</artifactId>
+ <version>9.0.71</version>
+ </dependency>
+
<!-- Spring Security -->
<dependency>
[kylin] 35/38: KYLIN-5537 Upgrade version, resolve vulnerability
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 111723d7a49df0832ee62fad271caa4a1b30e54a
Author: Yinghao Lin <yi...@kyligence.io>
AuthorDate: Tue Mar 7 14:48:03 2023 +0800
KYLIN-5537 Upgrade version, resolve vulnerability
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 00830ab3ba..b0159f2797 100644
--- a/pom.xml
+++ b/pom.xml
@@ -207,7 +207,7 @@
<esapi.version>2.3.0.0</esapi.version>
<api-util.version>1.0.2</api-util.version>
<neko-htmlunit.version>2.62.0</neko-htmlunit.version>
- <woodstox-core.version>5.3.0</woodstox-core.version>
+ <woodstox-core.version>5.4.0</woodstox-core.version>
<!--metric-->
<dropwizard.version>4.1.1</dropwizard.version>
[kylin] 12/38: [DIRTY] Rewrite spark InferFiltersFromConstraints
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 00e52f4b3f3da7715a6e0fbde34c9d5db7ff6e61
Author: Mingming Ge <7m...@gmail.com>
AuthorDate: Sun Jan 29 18:45:04 2023 +0800
[DIRTY] Rewrite spark InferFiltersFromConstraints
---
.../java/org/apache/kylin/common/KapConfig.java | 4 +
.../scala/org/apache/spark/sql/SparderEnv.scala | 31 +-
.../RewriteInferFiltersFromConstraints.scala | 101 +++++++
.../RewriteInferFiltersFromConstraintsSuite.scala | 320 +++++++++++++++++++++
4 files changed, 439 insertions(+), 17 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index c8aae82ddd..4f2132760e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -265,6 +265,10 @@ public class KapConfig {
return Integer.parseInt(config.getOptional("kylin.query.engine.spark-sql-shuffle-partitions", "-1"));
}
+ public Boolean isConstraintPropagationEnabled() {
+ return Boolean.parseBoolean(config.getOptional("kylin.query.engine.spark-constraint-propagation-enabled", FALSE));
+ }
+
/**
* LDAP filter
*/
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
index 41f6abe771..59ce599fce 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
@@ -23,13 +23,12 @@ import java.security.PrivilegedAction
import java.util.Map
import java.util.concurrent.locks.ReentrantLock
import java.util.concurrent.{Callable, ExecutorService}
-
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.security.UserGroupInformation
import org.apache.kylin.common.exception.{KylinException, KylinTimeoutException, ServerErrorCode}
import org.apache.kylin.common.msg.MsgPicker
import org.apache.kylin.common.util.{DefaultHostInfoFetcher, HadoopUtil, S3AUtil}
-import org.apache.kylin.common.{KylinConfig, QueryContext}
+import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
import org.apache.kylin.metadata.model.{NTableMetadataManager, TableExtDesc}
import org.apache.kylin.metadata.project.NProjectManager
import org.apache.kylin.query.runtime.plan.QueryToExecutionIDCache
@@ -39,7 +38,7 @@ import org.apache.spark.sql.KylinSession._
import org.apache.spark.sql.catalyst.optimizer.ConvertInnerJoinToSemiJoin
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.datasource.{KylinSourceStrategy, LayoutFileSourceStrategy}
+import org.apache.spark.sql.execution.datasource.{KylinSourceStrategy, LayoutFileSourceStrategy, RewriteInferFiltersFromConstraints}
import org.apache.spark.sql.execution.ui.PostQueryExecutionForKylin
import org.apache.spark.sql.hive.ReplaceLocationRule
import org.apache.spark.sql.udf.UdfManager
@@ -223,29 +222,17 @@ object SparderEnv extends Logging {
SparkSession.builder
.master("local")
.appName("sparder-local-sql-context")
- .withExtensions { ext =>
- ext.injectPlannerStrategy(_ => KylinSourceStrategy)
- ext.injectPlannerStrategy(_ => LayoutFileSourceStrategy)
- ext.injectPostHocResolutionRule(ReplaceLocationRule)
- ext.injectOptimizerRule(_ => new ConvertInnerJoinToSemiJoin())
- }
.enableHiveSupport()
.getOrCreateKylinSession()
case _ =>
SparkSession.builder
.appName(appName)
.master("yarn")
- //if user defined other master in kylin.properties,
- // it will get overwrite later in org.apache.spark.sql.KylinSession.KylinBuilder.initSparkConf
- .withExtensions { ext =>
- ext.injectPlannerStrategy(_ => KylinSourceStrategy)
- ext.injectPlannerStrategy(_ => LayoutFileSourceStrategy)
- ext.injectPostHocResolutionRule(ReplaceLocationRule)
- ext.injectOptimizerRule(_ => new ConvertInnerJoinToSemiJoin())
- }
.enableHiveSupport()
.getOrCreateKylinSession()
}
+
+ injectExtensions(sparkSession.extensions)
spark = sparkSession
logInfo("Spark context started successfully with stack trace:")
logInfo(Thread.currentThread().getStackTrace.mkString("\n"))
@@ -277,6 +264,16 @@ object SparderEnv extends Logging {
}
}
+ def injectExtensions(sse: SparkSessionExtensions): Unit = {
+ sse.injectPlannerStrategy(_ => KylinSourceStrategy)
+ sse.injectPlannerStrategy(_ => LayoutFileSourceStrategy)
+ sse.injectPostHocResolutionRule(ReplaceLocationRule)
+ sse.injectOptimizerRule(_ => new ConvertInnerJoinToSemiJoin())
+ if (KapConfig.getInstanceFromEnv.isConstraintPropagationEnabled) {
+ sse.injectOptimizerRule(_ => RewriteInferFiltersFromConstraints)
+ }
+ }
+
def registerListener(sc: SparkContext): Unit = {
val sparkListener = new SparkListener {
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraints.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraints.scala
new file mode 100644
index 0000000000..7d5132b744
--- /dev/null
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraints.scala
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasource
+
+import org.apache.spark.sql.catalyst.expressions.{And, Expression, ExpressionSet}
+import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints.{constructIsNotNullConstraints, inferAdditionalConstraints}
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.{FILTER, JOIN}
+
+object RewriteInferFiltersFromConstraints extends Rule[LogicalPlan] {
+
+ def apply(plan: LogicalPlan): LogicalPlan = {
+ if (conf.constraintPropagationEnabled) {
+ inferFilters(plan)
+ } else {
+ plan
+ }
+ }
+
+ private def inferFilters(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
+ _.containsAnyPattern(FILTER, JOIN)) {
+ case filter @ Filter(condition, child) =>
+ val newFilters = filter.constraints --
+ (child.constraints ++ splitConjunctivePredicates(condition))
+ if (newFilters.nonEmpty) {
+ Filter(And(newFilters.reduce(And), condition), child)
+ } else {
+ filter
+ }
+
+ case join @ Join(left, right, joinType, conditionOpt, _) =>
+ joinType match {
+ // For inner join, we can infer additional filters for both sides. LeftSemi is kind of an
+ // inner join, it just drops the right side in the final output.
+ case _: InnerLike | LeftSemi | LeftOuter =>
+ val allConstraints = getAllConstraints(left, right, conditionOpt)
+ val newLeft = inferNewFilter(left, allConstraints)
+ val newRight = inferNewFilter(right, allConstraints)
+ join.copy(left = newLeft, right = newRight)
+
+ // For right outer join, we can only infer additional filters for left side.
+ case RightOuter =>
+ val allConstraints = getAllConstraints(left, right, conditionOpt)
+ val newLeft = inferNewFilter(left, allConstraints)
+ join.copy(left = newLeft)
+
+ case LeftAnti =>
+ val allConstraints = getAllConstraints(left, right, conditionOpt)
+ val newRight = inferNewFilter(right, allConstraints)
+ join.copy(right = newRight)
+
+ case _ => join
+ }
+ }
+
+ private def getAllConstraints(
+ left: LogicalPlan,
+ right: LogicalPlan,
+ conditionOpt: Option[Expression]): ExpressionSet = {
+ val baseConstraints = left.constraints.union(right.constraints)
+ .union(ExpressionSet(conditionOpt.map(splitConjunctivePredicates).getOrElse(Nil)))
+ baseConstraints.union(inferAdditionalConstraints(baseConstraints))
+ }
+
+ private def inferNewFilter(plan: LogicalPlan, constraints: ExpressionSet): LogicalPlan = {
+ val newPredicates = constraints
+ .union(constructIsNotNullConstraints(constraints, plan.output))
+ .filter { c =>
+ c.references.nonEmpty && c.references.subsetOf(plan.outputSet) && c.deterministic
+ } -- plan.constraints
+ if (newPredicates.isEmpty) {
+ plan
+ } else {
+ Filter(newPredicates.reduce(And), plan)
+ }
+ }
+
+ protected def splitConjunctivePredicates(condition: Expression): Seq[Expression] = {
+ condition match {
+ case And(cond1, cond2) =>
+ splitConjunctivePredicates(cond1) ++ splitConjunctivePredicates(cond2)
+ case other => other :: Nil
+ }
+ }
+}
diff --git a/src/spark-project/spark-common/src/test/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraintsSuite.scala b/src/spark-project/spark-common/src/test/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraintsSuite.scala
new file mode 100644
index 0000000000..e766dd3b53
--- /dev/null
+++ b/src/spark-project/spark-common/src/test/scala/org/apache/spark/sql/execution/datasource/RewriteInferFiltersFromConstraintsSuite.scala
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasource
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.optimizer.{BooleanSimplification, CombineFilters, InferFiltersFromConstraints, PruneFilters, PushPredicateThroughJoin, PushPredicateThroughNonJoin, SimplifyBinaryComparison}
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{IntegerType, LongType}
+
+class RewriteInferFiltersFromConstraintsSuite extends PlanTest {
+
+ object Optimize extends RuleExecutor[LogicalPlan] {
+ val batches =
+ Batch("InferAndPushDownFilters", FixedPoint(100),
+ PushPredicateThroughJoin,
+ PushPredicateThroughNonJoin,
+ RewriteInferFiltersFromConstraints,
+ CombineFilters,
+ SimplifyBinaryComparison,
+ BooleanSimplification,
+ PruneFilters) :: Nil
+ }
+
+ val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
+
+ private def testConstraintsAfterJoin(
+ x: LogicalPlan,
+ y: LogicalPlan,
+ expectedLeft: LogicalPlan,
+ expectedRight: LogicalPlan,
+ joinType: JoinType,
+ condition: Option[Expression] = Some("x.a".attr === "y.a".attr)) = {
+ val originalQuery = x.join(y, joinType, condition).analyze
+ val correctAnswer = expectedLeft.join(expectedRight, joinType, condition).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("filter: filter out constraints in condition") {
+ val originalQuery = testRelation.where('a === 1 && 'a === 'b).analyze
+ val correctAnswer = testRelation
+ .where(IsNotNull('a) && IsNotNull('b) && 'a === 'b && 'a === 1 && 'b === 1).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("single inner join: filter out values on either side on equi-join keys") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val originalQuery = x.join(y,
+ condition = Some(("x.a".attr === "y.a".attr) && ("x.a".attr === 1) && ("y.c".attr > 5)))
+ .analyze
+ val left = x.where(IsNotNull('a) && "x.a".attr === 1)
+ val right = y.where(IsNotNull('a) && IsNotNull('c) && "y.c".attr > 5 && "y.a".attr === 1)
+ val correctAnswer = left.join(right, condition = Some("x.a".attr === "y.a".attr)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("single inner join: filter out nulls on either side on non equal keys") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val originalQuery = x.join(y,
+ condition = Some(("x.a".attr =!= "y.a".attr) && ("x.b".attr === 1) && ("y.c".attr > 5)))
+ .analyze
+ val left = x.where(IsNotNull('a) && IsNotNull('b) && "x.b".attr === 1)
+ val right = y.where(IsNotNull('a) && IsNotNull('c) && "y.c".attr > 5)
+ val correctAnswer = left.join(right, condition = Some("x.a".attr =!= "y.a".attr)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("single inner join with pre-existing filters: filter out values on either side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val originalQuery = x.where('b > 5).join(y.where('a === 10),
+ condition = Some("x.a".attr === "y.a".attr && "x.b".attr === "y.b".attr)).analyze
+ val left = x.where(IsNotNull('a) && 'a === 10 && IsNotNull('b) && 'b > 5)
+ val right = y.where(IsNotNull('a) && IsNotNull('b) && 'a === 10 && 'b > 5)
+ val correctAnswer = left.join(right,
+ condition = Some("x.a".attr === "y.a".attr && "x.b".attr === "y.b".attr)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("single outer join: no null filters are generated") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val originalQuery = x.join(y, FullOuter,
+ condition = Some("x.a".attr === "y.a".attr)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, originalQuery)
+ }
+
+ test("multiple inner joins: filter out values on all sides on equi-join keys") {
+ val t1 = testRelation.subquery('t1)
+ val t2 = testRelation.subquery('t2)
+ val t3 = testRelation.subquery('t3)
+ val t4 = testRelation.subquery('t4)
+
+ val originalQuery = t1.where('b > 5)
+ .join(t2, condition = Some("t1.b".attr === "t2.b".attr))
+ .join(t3, condition = Some("t2.b".attr === "t3.b".attr))
+ .join(t4, condition = Some("t3.b".attr === "t4.b".attr)).analyze
+ val correctAnswer = t1.where(IsNotNull('b) && 'b > 5)
+ .join(t2.where(IsNotNull('b) && 'b > 5), condition = Some("t1.b".attr === "t2.b".attr))
+ .join(t3.where(IsNotNull('b) && 'b > 5), condition = Some("t2.b".attr === "t3.b".attr))
+ .join(t4.where(IsNotNull('b) && 'b > 5), condition = Some("t3.b".attr === "t4.b".attr))
+ .analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("inner join with filter: filter out values on all sides on equi-join keys") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+
+ val originalQuery =
+ x.join(y, Inner, Some("x.a".attr === "y.a".attr)).where("x.a".attr > 5).analyze
+ val correctAnswer = x.where(IsNotNull('a) && 'a.attr > 5)
+ .join(y.where(IsNotNull('a) && 'a.attr > 5), Inner, Some("x.a".attr === "y.a".attr)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("inner join with alias: alias contains multiple attributes") {
+ val t1 = testRelation.subquery('t1)
+ val t2 = testRelation.subquery('t2)
+
+ val originalQuery = t1.select('a, Coalesce(Seq('a, 'b)).as('int_col)).as("t")
+ .join(t2, Inner, Some("t.a".attr === "t2.a".attr && "t.int_col".attr === "t2.a".attr))
+ .analyze
+ val correctAnswer = t1
+ .where(IsNotNull('a) && IsNotNull(Coalesce(Seq('a, 'b))) && 'a === Coalesce(Seq('a, 'b)))
+ .select('a, Coalesce(Seq('a, 'b)).as('int_col)).as("t")
+ .join(t2.where(IsNotNull('a)), Inner,
+ Some("t.a".attr === "t2.a".attr && "t.int_col".attr === "t2.a".attr))
+ .analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("inner join with alias: alias contains single attributes") {
+ val t1 = testRelation.subquery('t1)
+ val t2 = testRelation.subquery('t2)
+
+ val originalQuery = t1.select('a, 'b.as('d)).as("t")
+ .join(t2, Inner, Some("t.a".attr === "t2.a".attr && "t.d".attr === "t2.a".attr))
+ .analyze
+ val correctAnswer = t1
+ .where(IsNotNull('a) && IsNotNull('b) &&'a === 'b)
+ .select('a, 'b.as('d)).as("t")
+ .join(t2.where(IsNotNull('a)), Inner,
+ Some("t.a".attr === "t2.a".attr && "t.d".attr === "t2.a".attr))
+ .analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("generate correct filters for alias that don't produce recursive constraints") {
+ val t1 = testRelation.subquery('t1)
+
+ val originalQuery = t1.select('a.as('x), 'b.as('y)).where('x === 1 && 'x === 'y).analyze
+ val correctAnswer =
+ t1.where('a === 1 && 'b === 1 && 'a === 'b && IsNotNull('a) && IsNotNull('b))
+ .select('a.as('x), 'b.as('y)).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("No inferred filter when constraint propagation is disabled") {
+ withSQLConf(SQLConf.CONSTRAINT_PROPAGATION_ENABLED.key -> "false") {
+ val originalQuery = testRelation.where('a === 1 && 'a === 'b).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, originalQuery)
+ }
+ }
+
+ test("constraints should be inferred from aliased literals") {
+ val originalLeft = testRelation.subquery('left).as("left")
+ val optimizedLeft = testRelation.subquery('left).where(IsNotNull('a) && 'a <=> 2).as("left")
+
+ val right = Project(Seq(Literal(2).as("two")), testRelation.subquery('right)).as("right")
+ val condition = Some("left.a".attr === "right.two".attr)
+
+ val original = originalLeft.join(right, Inner, condition)
+ val correct = optimizedLeft.join(right, Inner, condition)
+
+ comparePlans(Optimize.execute(original.analyze), correct.analyze)
+ }
+
+ test("SPARK-23405: left-semi equal-join should filter out null join keys on both sides") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ testConstraintsAfterJoin(x, y, x.where(IsNotNull('a)), y.where(IsNotNull('a)), LeftSemi)
+ }
+
+ test("SPARK-21479: Outer join after-join filters push down to null-supplying side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val condition = Some("x.a".attr === "y.a".attr)
+ val originalQuery = x.join(y, LeftOuter, condition).where("x.a".attr === 2).analyze
+ val left = x.where(IsNotNull('a) && 'a === 2)
+ val right = y.where(IsNotNull('a) && 'a === 2)
+ val correctAnswer = left.join(right, LeftOuter, condition).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("SPARK-21479: Outer join pre-existing filters push down to null-supplying side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ val condition = Some("x.a".attr === "y.a".attr)
+ val originalQuery = x.join(y.where("y.a".attr > 5), RightOuter, condition).analyze
+ val left = x.where(IsNotNull('a) && 'a > 5)
+ val right = y.where(IsNotNull('a) && 'a > 5)
+ val correctAnswer = left.join(right, RightOuter, condition).analyze
+ val optimized = Optimize.execute(originalQuery)
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("SPARK-21479: Outer join no filter push down to preserved side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ testConstraintsAfterJoin(
+ x.where("a".attr === 1), y.where("a".attr === 1),
+ x.where(IsNotNull('a) && 'a === 1), y.where(IsNotNull('a) && 'a === 1),
+ LeftOuter)
+ }
+
+ test("SPARK-23564: left anti join should filter out null join keys on right side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ testConstraintsAfterJoin(x, y, x, y.where(IsNotNull('a)), LeftAnti)
+ }
+
+ test("SPARK-23564: left outer join should filter out null join keys on right side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ testConstraintsAfterJoin(x, y, x.where(IsNotNull('a)), y.where(IsNotNull('a)), LeftOuter)
+ }
+
+ test("SPARK-23564: right outer join should filter out null join keys on left side") {
+ val x = testRelation.subquery('x)
+ val y = testRelation.subquery('y)
+ testConstraintsAfterJoin(x, y, x.where(IsNotNull('a)), y, RightOuter)
+ }
+
+ test("Constraints should be inferred from cast equality constraint(filter higher data type)") {
+ val testRelation1 = LocalRelation('a.int)
+ val testRelation2 = LocalRelation('b.long)
+ val originalLeft = testRelation1.subquery('left)
+ val originalRight = testRelation2.where('b === 1L).subquery('right)
+
+ val left = testRelation1.where(IsNotNull('a) && 'a.cast(LongType) === 1L).subquery('left)
+ val right = testRelation2.where(IsNotNull('b) && 'b === 1L).subquery('right)
+
+ Seq(Some("left.a".attr.cast(LongType) === "right.b".attr),
+ Some("right.b".attr === "left.a".attr.cast(LongType))).foreach { condition =>
+ testConstraintsAfterJoin(originalLeft, originalRight, left, right, Inner, condition)
+ }
+
+ Seq(Some("left.a".attr === "right.b".attr.cast(IntegerType)),
+ Some("right.b".attr.cast(IntegerType) === "left.a".attr)).foreach { condition =>
+ testConstraintsAfterJoin(
+ originalLeft,
+ originalRight,
+ testRelation1.where(IsNotNull('a)).subquery('left),
+ right,
+ Inner,
+ condition)
+ }
+ }
+
+ test("Constraints shouldn't be inferred from cast equality constraint(filter lower data type)") {
+ val testRelation1 = LocalRelation('a.int)
+ val testRelation2 = LocalRelation('b.long)
+ val originalLeft = testRelation1.where('a === 1).subquery('left)
+ val originalRight = testRelation2.subquery('right)
+
+ val left = testRelation1.where(IsNotNull('a) && 'a === 1).subquery('left)
+ val right = testRelation2.where(IsNotNull('b)).subquery('right)
+
+ Seq(Some("left.a".attr.cast(LongType) === "right.b".attr),
+ Some("right.b".attr === "left.a".attr.cast(LongType))).foreach { condition =>
+ testConstraintsAfterJoin(originalLeft, originalRight, left, right, Inner, condition)
+ }
+
+ Seq(Some("left.a".attr === "right.b".attr.cast(IntegerType)),
+ Some("right.b".attr.cast(IntegerType) === "left.a".attr)).foreach { condition =>
+ testConstraintsAfterJoin(
+ originalLeft,
+ originalRight,
+ left,
+ testRelation2.where(IsNotNull('b) && 'b.attr.cast(IntegerType) === 1).subquery('right),
+ Inner,
+ condition)
+ }
+ }
+}
[kylin] 19/38: mirror: fix ut & compile
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2e83b6e8a592447a20f66860adeef7d622a709d9
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Tue Apr 25 21:20:55 2023 +0800
mirror: fix ut & compile
---
.../src/test/java/org/apache/kylin/rest/service/JobServiceTest.java | 2 +-
.../main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 27245e7046..c1fa88842b 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -1906,7 +1906,7 @@ public class JobServiceTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testJobSubdirectoryPermission() throws IOException {
+ public void testJobSubdirectoryPermission() throws IOException, PersistentException {
String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de";
NExecutableManager manager = NExecutableManager.getInstance(jobService.getConfig(), "default");
ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java
index 2fbf4652dc..0abb71ec21 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/RDSegmentBuildJob.java
@@ -72,6 +72,6 @@ public class RDSegmentBuildJob extends SegmentJob implements ResourceDetect {
private void writeCountDistinct() {
ResourceDetectUtils.write(new Path(rdSharedPath, ResourceDetectUtils.countDistinctSuffix()), //
- ResourceDetectUtils.findCountDistinctMeasure(readOnlyLayouts));
+ ResourceDetectUtils.findCountDistinctMeasure(getReadOnlyLayouts()));
}
}
[kylin] 38/38: [DIRTY] Fix Sonar
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1d1a9d0d607f259527a37b6962c4b7d0185c64b9
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Thu Apr 27 13:34:22 2023 +0800
[DIRTY] Fix Sonar
---
.../org/apache/kylin/common/util/StringHelper.java | 19 -------------------
1 file changed, 19 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java b/src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java
index 7be0bc9e07..33ef21727f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/StringHelper.java
@@ -181,25 +181,6 @@ public class StringHelper {
return r.toArray(new String[0]);
}
- // calculating length in UTF-8 of Java String without actually encoding it
- public static int utf8Length(CharSequence sequence) {
- int count = 0;
- for (int i = 0, len = sequence.length(); i < len; i++) {
- char ch = sequence.charAt(i);
- if (ch <= 0x7F) {
- count++;
- } else if (ch <= 0x7FF) {
- count += 2;
- } else if (Character.isHighSurrogate(ch)) {
- count += 4;
- ++i;
- } else {
- count += 3;
- }
- }
- return count;
- }
-
public static boolean equals(String a, String b) {
return a == null ? b == null : a.equals(b);
}