You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2024/04/12 08:24:03 UTC
(kylin) branch kylin5 updated (1d1ecbec63 -> 3f9b9c83be)
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from 1d1ecbec63 KYLIN-5823,fix text recognition bug
new d6285db6ab KYLIN-5792 Support ldap memberid mode
new 07354967d4 KYLIN-5793 Fix Chinese gibberish when Spring ${} attribute injection
new 91383f54d5 KYLIN-5795 Remove the check of `hadoop.tmp.dir`
new e824503eca KYLIN-5796 Support databinder auto grow collections limit
new 82ee25fdf1 KYLIN-5797 Remove nacos jar
new 4d3418c237 KYLIN-5798 Add user-table access cache for table-loading
new deb209f531 KYLIN-5799 Fix round, bround, lpad, rpad function
new dc344ce2ff KYLIN-5800 & KYLIN-5812 Support scalar subquery join aggregate pushdown
new e33ef57cce KYLIN-5791 & KYLIN-5801 Fix computation of timestampdff & add_month
new 9b8050e00a KYLIN-5802 Models with the same name exist when restore metadata
new bfff6d002c KYLIN-5803 Add file segments and smart cache
new 085d707216 KYLIN-5805 Refactor project lock
new a212350a51 KYLIN-5807 Fix query execute as user without data query acl
new c4aad6e78b KYLIN-5808 Optimize performance when saving model/adding CC
new 630a3aa7ce KYLIN-5809 Add TableIndex suggestion control in open api
new dcfa26067f KYLIN-5810 Add the spark.session.isDefined check in queryExec
new 5092e7a239 KYLIN-5811 Enhancement of SCD2
new ebd2f3d670 KYLIN-5814 Support lettuce sentinel mode
new 768a5ececc KYLIN-5815 Failed to add primaryKey of table when upgrading concurrently
new 2159699d8e KYLIN-5816 Add zk lock for dict v3
new 00352e8c2d KYLIN-5817 Query scanRows and bytes are incorrect
new c61dc41899 KYLIN-5828 Concurrently dict v2 jobs lead to abnormal encoding result
new f9255a515d KYLIN-5829 Add a tool for generating sql from dumped metadata
new 191d08c272 KYLIN-5830 Push columns of sortRel to subOlapContexts
new d7f37675e2 KYLIN-5832 Support queryDetect api
new 412eff5c73 KYLIN-5833 Use TableIndex answer select * only works for query
new e49c2f15a6 KYLIN-5836 Fix mapr env building error
new 148d8f61aa KYLIN-5837 Support only reusing user defined computedColumns
new eacb62b8aa KYLIN-5838 Replace join expression with the name of computedColumn
new be22ca0de7 KYLIN-5839 Block agg-push-down for non-equal-join
new 1c05436a01 KYLIN-5840 Fix vulnerability, Upgrade org.xerial.snappy:snappy-java to version 1.1.10.4
new 7d24ce642c KYLIN-5841 Fix vulnerability, Upgrade org.apache.avro:avro to version 1.11.3
new e804eee81e KYLIN-5844 & KYLIN-5843 Column ACL works for PushDown
new 0bbb74ee79 KYLIN-5845 Avoid model initialization for multi-thread
new 3f9b9c83be KYLIN-5846 upgrade spark version to 3.2.0-kylin-4.6.16.0
The 35 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build/bin/create-table.sh | 32 +
build/conf/kylin-tools-diag-log4j.xml | 6 +
build/conf/kylin-tools-log4j.xml | 6 +
.../kap/secondstorage/metadata/Manager.java | 23 +-
.../kap/secondstorage/metadata/NodeGroup.java | 5 +
.../kap/secondstorage/metadata/TablePlan.java | 5 +
pom.xml | 48 +-
.../kylin/rest/controller/NBasicController.java | 14 +
.../kylin/rest/controller/NSystemController.java | 33 +
src/common-service/pom.xml | 4 +
.../apache/kylin/rest/KylinPrepareEnvListener.java | 3 +-
.../kylin/rest/aspect/InsensitiveNameAspect.java | 3 +
.../kylin/rest/cache/AbstractKylinCache.java | 78 ++
.../apache/kylin/rest/cache/CacheConstant.java} | 12 +-
.../org/apache/kylin/rest/cache/RedisCache.java | 91 +--
.../org/apache/kylin/rest/cache/RedisCacheV2.java | 224 ++++++
.../apache/kylin/rest/config/AppInitializer.java | 21 +
.../config/initialize/EpochChangedListener.java | 16 +-
.../config/initialize/ModelBrokenListener.java | 12 +-
.../rest/request/SyncFileSegmentsRequest.java} | 23 +-
.../rest/response/SyncFileSegmentsResponse.java} | 29 +-
.../apache/kylin/rest/service/AccessService.java | 14 +-
.../kylin/rest/service/KylinUserService.java | 8 +-
.../apache/kylin/rest/service/ProjectService.java | 5 +-
.../apache/kylin/rest/service/SystemService.java | 44 +-
.../service/task/QueryHistoryTaskScheduler.java | 21 +-
.../task/RecommendationTopNUpdateScheduler.java | 16 +-
.../kylin/rest/cache/CacheConstantTest.java} | 13 +-
.../apache/kylin/rest/cache/RedisCacheTest.java} | 20 +-
.../apache/kylin/rest/cache/RedisCacheV2Test.java | 195 +++++
.../task/QueryHistoryTaskSchedulerTest.java | 17 +-
.../rest/util/InitResourceGroupUtilsTest.java} | 24 +-
src/core-common/pom.xml | 9 +
.../java/org/apache/kylin/common/KapConfig.java | 28 +-
.../org/apache/kylin/common/KylinConfigBase.java | 123 +++-
.../java/org/apache/kylin/common/QueryContext.java | 23 +-
.../apache/kylin/common/debug/BackdoorToggles.java | 5 -
.../kylin/common/exception/QueryErrorCode.java | 6 -
.../kylin/common/exception/ServerErrorCode.java | 2 +
.../common/exception/code/ErrorCodeServer.java | 9 +
.../common/exception/code/ErrorCodeSystem.java | 18 +-
.../kylin/common/exception/code/ErrorCodeTool.java | 3 +-
.../org/apache/kylin/common/msg/CnMessage.java | 10 +
.../java/org/apache/kylin/common/msg/Message.java | 9 +
.../common/persistence/InMemResourceStore.java | 7 +-
.../kylin/common/persistence/ResourceStore.java | 25 +-
.../common/persistence/RootPersistentEntity.java | 16 +-
.../persistence/ThreadViewResourceStore.java | 13 +-
.../persistence/lock/AbstractMemoryLock.java | 62 ++
.../persistence/lock/DeadLockException.java} | 9 +-
.../common/persistence/lock/DeadLockHandler.java} | 44 +-
.../common/persistence/lock/DeadLockInfo.java | 72 ++
.../common/persistence/lock/IntentionLock.java} | 8 +-
.../persistence/lock/IntentionReadWriteLock.java | 257 +++++++
.../persistence/lock/IntentionShareLock.java | 59 ++
.../persistence/lock/LockInterruptException.java} | 9 +-
.../common/persistence/lock/LockManager.java} | 27 +-
.../persistence/lock/LockManagerInstance.java} | 20 +-
.../persistence/lock/LockTimeoutException.java} | 9 +-
.../kylin/common/persistence/lock/MemoryLock.java} | 21 +-
.../common/persistence/lock/MemoryLockGraph.java | 171 +++++
.../common/persistence/lock/MemoryLockManager.java | 97 +++
.../common/persistence/lock/MemoryLockUtils.java | 206 ++++++
.../kylin/common/persistence/lock/ModuleLock.java} | 42 +-
.../common/persistence/lock/ModuleLockEnum.java | 87 +++
.../lock/OptimisticLockRetryException.java} | 7 +-
.../kylin/common/persistence/lock/PathLock.java} | 35 +-
.../common/persistence/lock/ProjectLock.java} | 36 +-
.../persistence/lock/ResourcePathParser.java | 121 ++++
.../kylin/common/persistence/lock/TempLock.java} | 7 +-
.../lock/TransactionDeadLockHandler.java | 168 +++++
.../common/persistence/lock/TransactionLock.java} | 11 +-
.../persistence/lock/TransactionLockManager.java} | 25 +-
.../persistence/lock/TransactionProjectLock.java} | 13 +-
.../persistence/lock/TransactionTempLock.java} | 12 +-
.../lock/rule/LockEliminationRule.java} | 24 +-
.../lock/rule/LockEliminationRuleExec.java | 57 ++
.../common/persistence/lock/rule/LockInfo.java | 49 ++
.../lock/rule/ModuleEliminationRule.java | 55 ++
.../persistence/lock/rule/PathEliminationRule.java | 70 ++
.../lock/rule/ProjectEliminationRule.java} | 25 +-
.../persistence/metadata/JdbcEpochStore.java | 20 +-
.../persistence/metadata/JdbcMetadataStore.java | 3 +
.../common/persistence/metadata/MetadataStore.java | 1 +
.../common/persistence/metadata/jdbc/JdbcUtil.java | 10 +
.../transaction/MessageSynchronization.java | 7 +-
.../persistence/transaction/TransactionLock.java | 96 ---
.../transaction/TransactionManagerInstance.java} | 33 +-
.../TransactionOptimisticLockManager.java} | 15 +-
.../TransactionPessimisticLockManager.java | 81 +++
.../common/persistence/transaction/UnitOfWork.java | 163 +++--
.../persistence/transaction/UnitOfWorkContext.java | 28 +-
.../persistence/transaction/UnitOfWorkParams.java | 24 +
.../org/apache/kylin/common/util/AddressUtil.java | 4 +
.../org/apache/kylin/common/util/ClassUtil.java | 25 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 9 +-
.../org/apache/kylin/common/util/JsonUtil.java | 8 +
.../org/apache/kylin/common/util/ThreadUtil.java | 3 +
.../resources/kylin_error_msg_conf_cn.properties | 13 +
.../resources/kylin_error_msg_conf_en.properties | 13 +
.../main/resources/kylin_errorcode_conf.properties | 13 +
.../resources/kylin_errorcode_conf_en.properties | 2 +
.../resources/kylin_errorcode_conf_zh.properties | 2 +
.../org/apache/kylin/common/KapConfigTest.java | 15 +-
.../apache/kylin/common/KylinConfigBaseTest.java | 30 +
.../persistence/JdbcAuditLogRecoveryTest.java | 24 +-
.../common/persistence/JdbcMetadataStoreTest.java | 45 +-
.../kylin/common/persistence/UnitOfWorkTest.java | 113 ++-
.../common/persistence/lock/CloseWaitTest.java | 167 +++++
.../persistence/lock/MemoryLockGraphTest.java | 218 ++++++
.../lock/rule/LockEliminationRuleExecTest.java | 159 ++++
.../metadata/JdbcAuditLogStoreTest.java | 8 +-
.../persistence/metadata/jdbc/JdbcUtilTest.java | 17 +
...ckTest.java => TransactionProjectLockTest.java} | 13 +-
.../apache/kylin/common/util/AddressUtilTest.java | 1 +
.../apache/kylin/common/util/ClassUtilTest.java | 63 ++
.../org/apache/kylin/job/common/SegmentUtil.java | 5 +-
.../org/apache/kylin/job/dao/JobStatistics.java | 10 +-
.../apache/kylin/job/dao/JobStatisticsBasic.java | 12 +-
.../apache/kylin/job/dao/JobStatisticsManager.java | 28 +-
.../org/apache/kylin/job/dao/NExecutableDao.java | 20 +-
.../kylin/job/execution/NExecutableManager.java | 16 +-
.../kylin/metadata/TempStatementManager.java | 10 +-
.../java/org/apache/kylin/metadata/acl/AclTCR.java | 17 +-
.../apache/kylin/metadata/acl/AclTCRManager.java | 21 +-
.../metadata/cachesync/CacheReloadChecker.java | 6 +-
.../kylin/metadata/cachesync/CachedCrudAssist.java | 53 +-
.../kylin/metadata/cube/cuboid/IndexMatcher.java | 7 +-
.../kylin/metadata/cube/model/IndexPlan.java | 10 +-
.../cube/model/NDataLoadingRangeManager.java | 111 +--
.../cube/model/NDataSegDetailsManager.java | 41 +-
.../kylin/metadata/cube/model/NDataSegment.java | 7 +-
.../kylin/metadata/cube/model/NDataflow.java | 5 +
.../metadata/cube/model/NDataflowManager.java | 100 +--
.../metadata/cube/model/NIndexPlanManager.java | 38 +-
.../kylin/metadata/favorite/AbstractAsyncTask.java | 11 +
.../metadata/favorite/AsyncAccelerationTask.java | 3 +-
.../kylin/metadata/favorite/AsyncTaskManager.java | 30 +-
.../kylin/metadata/favorite/FavoriteRule.java | 17 +-
.../metadata/favorite/FavoriteRuleManager.java | 22 +-
.../metadata/favorite/QueryHistoryIdOffset.java | 11 +
.../favorite/QueryHistoryIdOffsetManager.java | 25 +-
.../apache/kylin/metadata/jar/JarInfoManager.java | 34 +-
.../kylin/metadata/model/ColExcludedChecker.java | 13 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 5 +-
.../kylin/metadata/model/ComputedColumnDesc.java | 4 +
.../FilePartitionDesc.java} | 29 +-
.../apache/kylin/metadata/model/FusionModel.java | 10 +-
.../kylin/metadata/model/FusionModelManager.java | 11 +-
.../org/apache/kylin/metadata/model/JoinDesc.java | 30 +-
.../apache/kylin/metadata/model/NDataModel.java | 35 +-
.../kylin/metadata/model/NDataModelManager.java | 27 +-
.../metadata/model/NTableMetadataManager.java | 115 ++-
.../kylin/metadata/model/NonEquiJoinCondition.java | 301 +++++++-
.../apache/kylin/metadata/model/PartitionDesc.java | 36 +-
.../apache/kylin/metadata/model/SegmentRange.java | 86 +--
.../org/apache/kylin/metadata/model/Segments.java | 4 +
.../org/apache/kylin/metadata/model/TableDesc.java | 10 +-
.../apache/kylin/metadata/model/TableExtDesc.java | 24 +-
.../metadata/model/{tool => }/TruthTable.java | 184 +++--
.../metadata/model/schema/ImportModelContext.java | 2 +-
.../model/tool/JoinDescNonEquiCompBean.java | 60 --
.../model/tool/NonEquiJoinConditionComparator.java | 166 -----
.../model/tool/NonEquiJoinConditionVisitor.java | 59 --
.../metadata/model/tool/TypedLiteralConverter.java | 76 --
.../kylin/metadata/model/util/JoinDescUtil.java | 7 +-
.../metadata/model/util/scd2/SCD2CondChecker.java | 88 ++-
.../util/scd2/SCD2NonEquiCondSimplification.java | 201 ------
.../metadata/model/util/scd2/SCD2SqlConverter.java | 4 +-
.../metadata/model/util/scd2/Scd2Simplifier.java | 174 +++++
.../model/util/scd2/SimplifiedJoinDesc.java | 23 +-
.../kylin/metadata/project/NProjectManager.java | 22 +-
.../metadata/query/JdbcQueryHistoryStore.java | 8 +
.../metadata/query/NativeQueryRealization.java | 3 +
.../metadata/query/NoopJdbcQueryHistoryStore.java | 219 ++++++
.../kylin/metadata/query/QueryMetricsContext.java | 3 +-
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 6 +-
.../metadata/querymeta/SelectedColumnMeta.java | 1 -
.../resourcegroup/ResourceGroupManager.java | 2 +-
.../metadata/sourceusage/SourceUsageManager.java | 9 +-
.../metadata/streaming/DataParserManager.java | 20 +-
.../metadata/streaming/KafkaConfigManager.java | 17 +-
.../kylin/metadata/streaming/StreamingManager.java | 13 +-
.../metadata/upgrade/GlobalAclVersionManager.java | 10 +-
.../apache/kylin/metadata/user/ManagedUser.java | 7 +
.../kylin/metadata/user/NKylinUserManager.java | 37 +-
.../metadata/usergroup/NUserGroupManager.java | 4 +-
.../apache/kylin/metadata/usergroup/UserGroup.java | 7 +
.../kylin/query/util/QueryInterruptChecker.java | 23 +-
.../org/apache/kylin/rest/security/AclManager.java | 18 +-
.../org/apache/kylin/rest/security/AclRecord.java | 2 +-
.../security/PasswordPlaceholderConfigurer.java | 1 +
.../org/apache/kylin/rest/security/UserAcl.java | 7 +
.../apache/kylin/rest/security/UserAclManager.java | 4 +-
.../apache/kylin/rest/util/AclPermissionUtil.java | 2 +-
.../org/apache/kylin/util/BrokenEntityProxy.java | 10 +-
.../metadata/cachesync/CachedCrudAssistTest.java | 63 ++
.../apache/kylin/metadata/cube/IndexPlanTest.java | 49 +-
.../cube/model/NDataLoadingRangeManagerTest.java | 28 +-
.../cube/model/NDataSegDetailsManagerTest.java | 2 +-
.../metadata/epoch/EnhancedUnitOfWorkTest.java | 8 +-
.../kylin/metadata/model/ModelNonEquiCondMock.java | 8 +-
.../metadata/model/NDataModelManagerTest.java | 50 ++
.../metadata/model/NeqEquiJoinConditionTest.java | 281 ++++++++
.../tool/NonEquiJoinConditionComparatorTest.java | 234 ------
.../metadata/project/NProjectManagerTest.java | 32 +
.../org/apache/kylin/model/NDataModelTest.java | 53 ++
.../rest/controller/v2/SegmentControllerV2.java | 60 +-
.../kylin/rest/response/ExecutableResponse.java | 4 +-
.../kylin/rest/service/ModelBuildService.java | 28 +-
.../sdk/datasource/PushdownRunnerSDKImplTest.java | 10 +-
.../apache/kylin/rest/service/TableExtService.java | 2 +-
.../apache/kylin/rest/service/TableService.java | 127 ++--
.../apache/kylin/rest/service/SparkDDLTest.java | 26 +-
.../rest/service/StreamingTableServiceTest.java | 4 -
.../e78a89dd-847f-4574-8afa-8768b4228b72.json | 1 -
.../e78a89dd-847f-4574-8afa-8768b4228b73.json | 1 -
.../e78a89dd-847f-4574-8afa-8768b4228b74.json | 1 -
.../org/apache/kylin/newten/AutoMergeTest.java | 7 +-
.../routing/DataflowCapabilityCheckerTest.java | 3 +-
.../query/routing/RealizationChooserTest.java | 24 +-
.../kylin/query/rules/AggPushdownRuleTest.java | 4 +-
.../kylin/query/rules/CalciteRuleTestBase.java | 12 +-
.../query/rules/ScalarSubqueryJoinRuleTest.java | 112 +++
.../org/apache/kylin/util/OlapContextTestUtil.java | 19 +-
.../query/rules/ScalarSubqueryJoinRuleTest.xml | 778 ++++++++++++++++++++
.../query/sql_scalar_subquery/query01.sql | 26 +
.../query/sql_scalar_subquery/query02.sql | 30 +
.../query/sql_scalar_subquery/query03.sql | 32 +
.../query/sql_scalar_subquery/query04.sql | 28 +
.../query/sql_scalar_subquery/query05.sql | 37 +
.../query/sql_scalar_subquery/query06.sql | 35 +
.../query/sql_scalar_subquery/query07.sql | 33 +
.../query/sql_scalar_subquery/query08.sql | 36 +
.../query/sql_scalar_subquery/query09.sql | 36 +
.../query/sql_scalar_subquery/query10.sql | 33 +
.../query/sql_scalar_subquery/query11.sql | 32 +
.../query/sql_scalar_subquery/query12.sql | 41 ++
.../query/sql_scalar_subquery/query13.sql | 36 +
.../query/sql_scalar_subquery/query14.sql | 32 +
.../query/sql_scalar_subquery/query15.sql | 27 +
.../query/sql_scalar_subquery/query16.sql | 33 +
.../query/sql_scalar_subquery/query17.sql | 31 +
.../query/sql_scalar_subquery/query18.sql | 65 ++
.../query/sql_scalar_subquery/query19.sql | 26 +
.../org/apache/kylin/event/HAMetadataTest.java | 23 +-
src/metadata-server/pom.xml | 5 +
.../kylin/rest/controller/NTableController.java | 5 +-
.../controller/NProjectControllerKylinTest.java | 17 +-
.../rest/controller/NProjectControllerV2Test.java | 17 +-
.../rest/controller/NUserGroupControllerTest.java | 38 +-
.../controller/NUserGroupControllerV2Test.java | 22 +-
.../rest/controller/OpenAclTCRControllerTest.java | 15 +-
.../controller/open/OpenAclTCRControllerTest.java | 8 +-
.../open/OpenUserGroupControllerTest.java | 14 +-
.../apache/kylin/rest/request/ModelRequest.java | 4 +
.../rest/response/LayoutRecDetailResponse.java | 4 +-
.../kylin/rest/service/ModelSemanticHelper.java | 58 +-
.../apache/kylin/rest/service/ModelService.java | 213 +++---
.../rest/service/params/BasicSegmentParams.java | 3 +-
.../params/IncrementBuildSegmentParams.java | 10 +-
.../rest/util/SCD2SimplificationConvertUtil.java | 30 +-
.../kylin/rest/request/ModelRequestTest.java | 6 +-
.../apache/kylin/rest/service/BaseIndexTest.java | 7 +-
.../kylin/rest/service/IndexPlanServiceTest.java | 13 +-
.../rest/service/ModelServiceBrokenRepairTest.java | 1 +
.../service/ModelServiceSemanticUpdateTest.java | 456 ++++++------
.../kylin/rest/service/ModelServiceTest.java | 34 +-
.../service/ModelTdsServiceColumnNameTest.java | 2 +-
.../kylin/rest/service/ModelTdsServiceTest.java | 13 +-
.../kylin/rest/service/TableReloadServiceTest.java | 8 +-
.../kylin/rest/service/TableServiceTest.java | 6 +-
.../apache/kylin/query/relnode/ContextUtil.java | 24 +-
.../kylin/query/relnode/KapNonEquiJoinRel.java | 30 +-
.../org/apache/kylin/query/relnode/KapSortRel.java | 19 +-
.../apache/kylin/query/relnode/KapTableScan.java | 10 +-
.../apache/kylin/query/relnode/KapValuesRel.java | 21 +-
.../apache/kylin/query/relnode/OLAPContext.java | 19 +-
.../apache/kylin/query/relnode/OLAPValuesRel.java | 13 +-
.../kylin/query/routing/QueryLayoutChooser.java | 7 +-
.../kylin/query/routing/RealizationChooser.java | 4 +-
.../security/HackSelectStarWithColumnACL.java | 259 ++++---
.../kylin/query/util/ComputedColumnRewriter.java | 17 +-
.../kylin/query/util/ConvertToComputedColumn.java | 80 ++-
.../org/apache/kylin/query/util/PushDownUtil.java | 79 +-
.../query/util/RestoreFromComputedColumn.java | 19 +-
.../kylin/rest/controller/NQueryController.java | 109 ++-
.../rest/controller/NQueryControllerTest.java | 27 +
.../rest/controller/NQueryMetaControllerTest.java | 2 +
.../rest/controller/SparkSourceControllerTest.java | 12 +-
src/query-service/pom.xml | 5 +
.../kylin/rest/request/QueryDetectRequest.java} | 17 +-
.../kylin/rest/response/QueryDetectResponse.java | 156 ++++
.../apache/kylin/rest/response/SQLResponse.java | 44 +-
.../apache/kylin/rest/response/SQLResponseV2.java | 19 +
.../kylin/rest/service/QueryCacheManager.java | 113 ++-
.../apache/kylin/rest/service/QueryService.java | 84 ++-
.../kylin/rest/service/QueryCacheManagerTest.java | 191 +++++
.../kylin/rest/service/QueryServiceTest.java | 302 +++++++-
.../rest/util/QueryCacheSignatureUtilTest.java | 16 +-
.../kyligence/kap/query/optrule/KAPValuesRule.java | 11 +-
.../kyligence/kap/query/optrule/KapJoinRule.java | 242 +++++--
.../kap/query/optrule/ScalarSubqueryJoinRule.java | 799 +++++++++++++++++++++
.../apache/kylin/query/blacklist/SQLBlacklist.java | 13 +-
.../kylin/query/blacklist/SQLBlacklistManager.java | 28 +-
.../apache/kylin/query/engine/PlannerFactory.java | 12 +-
.../org/apache/kylin/query/engine/QueryExec.java | 44 +-
.../kylin/query/engine/QueryRoutingEngine.java | 7 +
.../engine/exec/calcite/CalciteQueryPlanExec.java | 3 +
.../engine/exec/sparder/SparderQueryPlanExec.java | 19 +-
.../org/apache/kylin/query/udf/SparkMathUDF.java | 7 +-
.../org/apache/kylin/query/udf/SparkStringUDF.java | 15 +-
.../java/org/apache/kylin/query/util/HepUtils.java | 20 +-
.../kylin/query/util/QueryContextCutter.java | 24 +-
.../kylin/query/engine/QueryRoutingEngineTest.java | 37 +-
.../kylin/query/engine/SelectRealizationTest.java | 6 +-
.../security/HackSelectStarWithColumnACLTest.java | 405 +++++++----
.../apache/kylin/query/util/CCOnRealModelTest.java | 68 +-
.../apache/kylin/query/util/PushDownUtilTest.java | 359 +++++----
src/server/pom.xml | 2 +
.../org/apache/kylin/rest/QueryNodeFilter.java | 4 +
.../rest/health/MetaStoreHealthIndicator.java | 3 +-
.../engine/spark/application/SparkApplication.java | 16 +-
.../kylin/engine/spark/job/MppOnTheFlyImpl.java | 160 +++++
.../kylin/engine/spark/job/NSparkExecutable.java | 28 +-
.../engine/spark/job/NSparkUpdateMetadataStep.java | 6 +-
.../merger/AfterMergeOrRefreshResourceMerger.java | 3 +-
.../spark/source/NSparkMetadataExplorer.java | 25 +-
.../engine/spark/utils/ComputedColumnEvalUtil.java | 20 +-
.../engine/spark/builder/CreateFlatTable.scala | 15 +-
.../engine/spark/builder/DFBuilderHelper.scala | 6 +
.../engine/spark/builder/DFDictionaryBuilder.scala | 33 +-
.../engine/spark/builder/DFTableEncoder.scala | 13 +-
.../spark/builder/DictionaryBuilderHelper.java | 5 +-
.../spark/builder/SnapshotPartitionBuilder.scala | 6 +-
.../kylin/engine/spark/builder/ZKHelper.scala | 42 ++
.../spark/builder/v3dict/DictionaryBuilder.scala | 69 +-
.../kylin/engine/spark/job/FiltersUtil.scala | 2 +-
.../kylin/engine/spark/job/FlatTableHelper.scala | 2 +-
.../kylin/engine/spark/job/SegmentBuildJob.java | 7 +-
.../apache/kylin/engine/spark/job/SegmentJob.java | 2 +-
.../engine/spark/job/stage/build/BuildStage.scala | 2 +-
.../job/stage/build/FlatTableAndDictBase.scala | 77 +-
.../stage/build/MaterializedFactTableView.scala | 32 +-
.../spark/smarter/IndexDependencyParser.scala | 60 +-
.../engine/spark/dict/NGlobalDictionaryV2Test.java | 77 +-
.../kylin/engine/spark/job/JobManagerTest.java | 5 +-
.../engine/spark/job/NSparkMergingJobTest.java | 10 +-
.../spark/source/NSparkMetadataExplorerTest.java | 14 +-
.../kylin/engine/spark/builder/TestDFChooser.scala | 5 +-
.../engine/spark/builder/TestGlobalDictBuild.scala | 58 +-
.../v3dict/GlobalDictionaryUpdateSuite.scala | 6 +-
.../kylin-soft-affinity-cache-hadoop3/pom.xml | 81 +++
.../cache/kylin/KylinCacheFileSystemHadoop3.java | 43 ++
.../kylin-soft-affinity-cache/pom.xml | 26 +
.../kylin/cache/fs/AbstractCacheFileSystem.java | 277 ++++---
.../kylin/cache/fs/AlluxioHdfsFileInputStream.java | 10 +-
.../kylin/cache/fs/CacheFileSystemConstants.java | 32 +-
.../kylin/cache/fs/ManagerOfCacheFileContent.java | 116 +++
.../kylin/cache/fs/ManagerOfCacheFileStatus.java | 156 ++++
.../kylin/cache/kylin/KylinCacheFileSystem.java | 123 +++-
.../cache/kylin/OnlyForTestCacheFileSystem.java | 16 +-
.../softaffinity/SoftAffinityBookKeeping.java | 80 +++
.../cache/softaffinity/SoftAffinityConstants.java | 13 +-
.../softaffinity/SoftAffinityTextInputFormat.java | 115 +++
.../apache/kylin/cache/utils/Hadoop3CompaUtil.java | 164 +++++
.../apache/kylin/cache/utils/ReflectionUtil.java | 5 +-
.../org/apache/kylin/fileseg/FileSegments.java | 612 ++++++++++++++++
.../apache/kylin/fileseg/FileSegmentsDetector.java | 136 ++++
.../hive/serde2/lazy/LazyQuoteAwareSerDe.java | 795 ++++++++++++++++++++
.../kylin/softaffinity/SoftAffinityManager.scala | 38 +-
.../execution/datasources/CacheFilePartition.scala | 11 +-
.../datasources/CachePartitionedFile.scala | 9 +-
.../kylin/cache/fs/KylinCacheFileSystemTest.java | 135 ----
.../cache/fs/ManagerOfCacheFileContentTest.java | 74 ++
.../cache/fs/ManagerOfCacheFileStatusTest.java | 69 ++
.../cache/kylin/KylinCacheFileSystemTest.java | 203 ++++++
.../serde2/lazy/lazy/LazyQuoteAwareSerDeTest.java | 290 ++++++++
.../test/resources/default_3ywc6z78.emp/emp.csv | 3 +
.../resources/default_3ywc6z78.emp/emp_data_1.csv | 4 +
.../src/test/resources/log4j2.xml | 24 +
.../kylin/KylinCacheFileSystemWithSparkTest.scala | 140 ++++
.../HiveSoftAffinityAndLocalCacheTest.scala | 213 ++++++
src/spark-project/sparder/pom.xml | 8 +
.../query/plugin/runtime/MppOnTheFlyProvider.java | 47 ++
.../kylin/query/pushdown/SparkSqlClient.scala | 51 +-
.../kylin/query/runtime/ExpressionConverter.scala | 25 +-
.../kylin/query/runtime/SparderRexVisitor.scala | 8 +-
.../kylin/query/runtime/plan/TableScanPlan.scala | 53 +-
.../kylin/query/util/SparderDerivedUtil.scala | 12 +-
.../scala/org/apache/spark/sql/KapFunctions.scala | 4 +-
.../scala/org/apache/spark/sql/SparderEnv.scala | 6 +-
.../sql/catalyst/expressions/KapExpresssions.scala | 24 +-
.../sql/hive/KylinHiveSessionStateBuilder.scala | 60 +-
.../apache/spark/sql/hive/QueryMetricUtils.scala | 4 +-
.../org/apache/spark/sql/udf/DictEncodeImpl.scala | 9 +-
.../query/pushdown/PushDownRunnerJdbcImplTest.java | 56 +-
.../sql/execution/SparkQueryMetricUtilsSuite.scala | 4 +-
.../spark/sql/hive/ReplaceLocationRuleTest.scala | 12 +-
.../org/apache/spark/dict/NBucketDictionary.java | 62 +-
.../spark/dict/NGlobalDictBuilderAssist.scala | 5 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java | 10 +-
.../org/apache/spark/dict/NGlobalDictS3Store.java | 16 +-
.../org/apache/spark/dict/NGlobalDictStore.java | 4 +-
.../org/apache/spark/dict/NGlobalDictionaryV2.java | 107 ++-
.../sql/execution/datasource/FilePruner.scala | 4 +
.../streaming/manager/StreamingJobManager.java | 9 +-
.../apache/kylin/helper/MetadataToolHelper.java | 139 +++-
.../org/apache/kylin/tool/MaintainModeTool.java | 4 +-
.../apache/kylin/tool/garbage/StorageCleaner.java | 22 +-
.../kylin/tool/security/KylinPasswordResetCLI.java | 10 +-
.../kylin/tool/upgrade/UpdateProjectCLI.java | 29 +-
.../kylin/tool}/util/CreateTableFromJson.java | 31 +-
.../java/org/apache/kylin/tool/util/LdapUtils.java | 12 +-
.../util/ProjectTemporaryTableCleanerHelper.java | 4 +-
.../org/apache/kylin/tool/AuditLogToolTest.java | 11 +-
.../org/apache/kylin/tool/MetadataToolTest.java | 57 ++
.../apache/kylin/tool/MetadataToolTestFixture.java | 2 +-
.../apache/kylin/tool/YarnApplicationToolTest.java | 11 +-
.../kylin/tool/routine/MaintainModeToolTest.java | 40 +-
.../tool/security/KylinPasswordResetCLITest.java | 11 +-
.../kylin/tool/util/CreateTableFromJsonTest.java | 68 ++
.../org/apache/kylin/tool/util/LdapUtilsTest.java | 57 ++
.../ProjectTemporaryTableCleanerHelperTest.java | 5 +-
424 files changed, 16899 insertions(+), 4212 deletions(-)
create mode 100755 build/bin/create-table.sh
create mode 100644 src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => common-service/src/main/java/org/apache/kylin/rest/cache/CacheConstant.java} (73%)
create mode 100644 src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCacheV2.java
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => common-service/src/main/java/org/apache/kylin/rest/request/SyncFileSegmentsRequest.java} (68%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java => common-service/src/main/java/org/apache/kylin/rest/response/SyncFileSegmentsResponse.java} (68%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => common-service/src/test/java/org/apache/kylin/rest/cache/CacheConstantTest.java} (71%)
copy src/{core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java => common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheTest.java} (64%)
create mode 100644 src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheV2Test.java
copy src/{core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java => common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java} (56%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java} (82%)
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java} (50%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java} (82%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java} (80%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SimplifiedJoinDesc.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java} (51%)
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java} (68%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java} (81%)
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java} (69%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java} (51%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java} (82%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java} (59%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java} (58%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java} (82%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java} (72%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SimplifiedJoinDesc.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java} (52%)
copy src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java} (74%)
rename src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java} (73%)
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java} (65%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java} (59%)
delete mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java
copy src/core-common/src/main/java/org/apache/kylin/common/{util/ThreadUtil.java => persistence/transaction/TransactionManagerInstance.java} (56%)
copy src/{spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java => core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java} (68%)
create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java
create mode 100644 src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java
create mode 100644 src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java
create mode 100644 src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java
rename src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/{TransactionLockTest.java => TransactionProjectLockTest.java} (89%)
copy src/core-metadata/src/main/java/org/apache/kylin/metadata/{usergroup/UserGroup.java => model/FilePartitionDesc.java} (71%)
rename src/core-metadata/src/main/java/org/apache/kylin/metadata/model/{tool => }/TruthTable.java (54%)
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/JoinDescNonEquiCompBean.java
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparator.java
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionVisitor.java
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TypedLiteralConverter.java
delete mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2NonEquiCondSimplification.java
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/Scd2Simplifier.java
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NoopJdbcQueryHistoryStore.java
create mode 100644 src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
create mode 100644 src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NeqEquiJoinConditionTest.java
delete mode 100644 src/core-metadata/src/test/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparatorTest.java
create mode 100644 src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java
create mode 100644 src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java
create mode 100644 src/kylin-it/src/test/resources/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.xml
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query01.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query02.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query03.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query04.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query05.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query06.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query07.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query08.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query09.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query10.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query11.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query12.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query13.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query14.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query15.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query16.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query17.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query18.sql
create mode 100644 src/kylin-it/src/test/resources/query/sql_scalar_subquery/query19.sql
rename src/{core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java => query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java} (71%)
create mode 100644 src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java
create mode 100644 src/query-service/src/test/java/org/apache/kylin/rest/service/QueryCacheManagerTest.java
create mode 100644 src/query/src/main/java/io/kyligence/kap/query/optrule/ScalarSubqueryJoinRule.java
create mode 100644 src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/MppOnTheFlyImpl.java
create mode 100644 src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala
create mode 100644 src/spark-project/kylin-soft-affinity-cache-hadoop3/pom.xml
create mode 100644 src/spark-project/kylin-soft-affinity-cache-hadoop3/src/main/java/io/kyligence/kap/cache/kylin/KylinCacheFileSystemHadoop3.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContent.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatus.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityBookKeeping.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityTextInputFormat.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/Hadoop3CompaUtil.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegments.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegmentsDetector.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/hive/serde2/lazy/LazyQuoteAwareSerDe.java
delete mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/KylinCacheFileSystemTest.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContentTest.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatusTest.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/kylin/KylinCacheFileSystemTest.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/hive/serde2/lazy/lazy/LazyQuoteAwareSerDeTest.java
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp.csv
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp_data_1.csv
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/resources/log4j2.xml
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/kylin/cache/kylin/KylinCacheFileSystemWithSparkTest.scala
create mode 100644 src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/spark/sql/hive/execution/HiveSoftAffinityAndLocalCacheTest.scala
create mode 100644 src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/runtime/MppOnTheFlyProvider.java
rename src/{core-metadata/src/main/java/org/apache/kylin/rest => tool/src/main/java/org/apache/kylin/tool}/util/CreateTableFromJson.java (81%)
create mode 100644 src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java
create mode 100644 src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java
(kylin) 28/35: KYLIN-5837 Support only reusing user defined computedColumns
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 148d8f61aaafbca0c207d76f0747a85ed0b11fbb
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Mon Sep 11 14:43:04 2023 +0800
KYLIN-5837 Support only reusing user defined computedColumns
---
.../java/org/apache/kylin/common/KapConfig.java | 27 +++-------
.../org/apache/kylin/common/KylinConfigBase.java | 25 ++++++---
.../org/apache/kylin/common/KapConfigTest.java | 15 ++----
.../apache/kylin/common/KylinConfigBaseTest.java | 8 +++
.../kylin/query/util/ComputedColumnRewriter.java | 17 +++---
.../kylin/query/util/ConvertToComputedColumn.java | 60 ++++++++--------------
.../query/util/RestoreFromComputedColumn.java | 19 +++----
7 files changed, 79 insertions(+), 92 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index c7c10c500d..c79a588c95 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -22,13 +22,14 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
-import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.annotation.Clarification;
import org.apache.kylin.common.util.EncryptUtil;
import org.apache.kylin.common.util.FileUtils;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
@Clarification(priority = Clarification.Priority.MAJOR, msg = "Enterprise")
public class KapConfig {
@@ -233,21 +234,6 @@ public class KapConfig {
return config.getOptional("kylin.smart.conf." + conf, null);
}
- /**
- * Query
- */
- public boolean isImplicitComputedColumnConvertEnabled() {
- return Boolean.parseBoolean(config.getOptional("kylin.query.implicit-computed-column-convert", "true"));
- }
-
- public boolean isAggComputedColumnRewriteEnabled() {
- return Boolean.parseBoolean(config.getOptional("kylin.query.agg-computed-column-rewrite", "true"));
- }
-
- public int getComputedColumnMaxRecursionTimes() {
- return Integer.parseInt(config.getOptional("kylin.query.computed-column-max-recursion-times", "10"));
- }
-
public boolean isJdbcEscapeEnabled() {
return Boolean.parseBoolean(config.getOptional("kylin.query.jdbc-escape-enabled", "true"));
}
@@ -266,7 +252,8 @@ public class KapConfig {
}
public Boolean isConstraintPropagationEnabled() {
- return Boolean.parseBoolean(config.getOptional("kylin.query.engine.spark-constraint-propagation-enabled", FALSE));
+ return Boolean
+ .parseBoolean(config.getOptional("kylin.query.engine.spark-constraint-propagation-enabled", FALSE));
}
/**
@@ -716,11 +703,13 @@ public class KapConfig {
}
public boolean isApplyLimitInfoToSourceScanRowsEnabled() {
- return Boolean.parseBoolean(config.getOptional("kylin.query.apply-limit-info-to-source-scan-rows-enabled", FALSE));
+ return Boolean
+ .parseBoolean(config.getOptional("kylin.query.apply-limit-info-to-source-scan-rows-enabled", FALSE));
}
public boolean isAutoAdjustBigQueryRowsThresholdEnabled() {
- return Boolean.parseBoolean(config.getOptional("kylin.query.auto-adjust-big-query-rows-threshold-enabled", FALSE));
+ return Boolean
+ .parseBoolean(config.getOptional("kylin.query.auto-adjust-big-query-rows-threshold-enabled", FALSE));
}
public long getBigQuerySecond() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a2518358a9..3042b23c58 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -793,6 +793,18 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.metadata.only-reuse-user-defined-computed-column", FALSE));
}
+ public boolean isConvertExpressionToCcEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.implicit-computed-column-convert", TRUE));
+ }
+
+ public boolean isAggComputedColumnRewriteEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.agg-computed-column-rewrite", TRUE));
+ }
+
+ public int getConvertCcMaxIterations() {
+ return Integer.parseInt(getOptional("kylin.query.computed-column-max-recursion-times", "10"));
+ }
+
/**
* expose computed column in the table metadata and select * queries
*/
@@ -1941,14 +1953,14 @@ public abstract class KylinConfigBase implements Serializable {
}
public String getMppOnTheFlyLayoutsProvider() {
- return this.getOptional("kylin.query.mpp-on-the-fly-layouts-provider", "io.kyligence.kap.engine.spark.job.MppOnTheFlyImpl");
+ return this.getOptional("kylin.query.mpp-on-the-fly-layouts-provider",
+ "io.kyligence.kap.engine.spark.job.MppOnTheFlyImpl");
}
public boolean enableReplaceDynamicParams() {
return Boolean.parseBoolean(this.getOptional("kylin.query.replace-dynamic-params-enabled", FALSE));
}
-
public String getPercentileApproxAlgorithm() {
// Valid values: t-digest
return this.getOptional("kylin.query.percentile-approx-algorithm", "");
@@ -2870,8 +2882,7 @@ public abstract class KylinConfigBase implements Serializable {
}
public long getTableAccessCacheTTL() {
- return TimeUtil.timeStringAs(getOptional("kylin.source.hive.table-access-cache-ttl", "7d"),
- TimeUnit.MINUTES);
+ return TimeUtil.timeStringAs(getOptional("kylin.source.hive.table-access-cache-ttl", "7d"), TimeUnit.MINUTES);
}
public String[] getHiveDatabases() {
@@ -4126,11 +4137,13 @@ public abstract class KylinConfigBase implements Serializable {
}
public boolean isKylinLocalCacheEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.storage.columnar.spark-conf.spark.kylin.local-cache.enabled", FALSE));
+ return Boolean
+ .parseBoolean(getOptional("kylin.storage.columnar.spark-conf.spark.kylin.local-cache.enabled", FALSE));
}
public boolean isKylinFileStatusCacheEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.storage.columnar.spark-conf.spark.hadoop.spark.kylin.file-status-cache.enabled", FALSE));
+ return Boolean.parseBoolean(getOptional(
+ "kylin.storage.columnar.spark-conf.spark.hadoop.spark.kylin.file-status-cache.enabled", FALSE));
}
public boolean isSupportPushdownHiveCsvEnhancement() {
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KapConfigTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KapConfigTest.java
index ba98e66eab..ed20497851 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KapConfigTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KapConfigTest.java
@@ -70,15 +70,6 @@ public class KapConfigTest {
map.put("getChannelUser", new PropertiesEntity("kylin.env.channel", "on-premises", "on-premises"));
- map.put("isImplicitComputedColumnConvertEnabled",
- new PropertiesEntity("kylin.query.implicit-computed-column-convert", "true", true));
-
- map.put("isAggComputedColumnRewriteEnabled",
- new PropertiesEntity("kylin.query.agg-computed-column-rewrite", "true", true));
-
- map.put("getComputedColumnMaxRecursionTimes",
- new PropertiesEntity("kylin.query.computed-column-max-recursion-times", "10", 10));
-
map.put("isJdbcEscapeEnabled", new PropertiesEntity("kylin.query.jdbc-escape-enabled", "true", true));
map.put("getListenerBusBusyThreshold",
@@ -258,8 +249,10 @@ public class KapConfigTest {
new PropertiesEntity("kylin.query.spark-job-trace-parallel-max", "50", 50));
map.put("isSourceUsageUnwrapComputedColumn",
new PropertiesEntity("kylin.metadata.history-source-usage-unwrap-computed-column", "true", true));
- map.put("getKerberosTGTRenewalInterval", new PropertiesEntity("kylin.kerberos.tgt-renewal-interval-minutes", "10", 10L));
- map.put("getKerberosTGTRetryInterval", new PropertiesEntity("kylin.kerberos.tgt-retry-interval-minutes", "60", 60L));
+ map.put("getKerberosTGTRenewalInterval",
+ new PropertiesEntity("kylin.kerberos.tgt-renewal-interval-minutes", "10", 10L));
+ map.put("getKerberosTGTRetryInterval",
+ new PropertiesEntity("kylin.kerberos.tgt-retry-interval-minutes", "60", 60L));
}
@Test
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 59e4a3fc9b..f01a4ab8a1 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -105,6 +105,14 @@ class KylinConfigBaseTest {
new PropertiesEntity("kylin.metrics.system-cube-sink-default-class",
"org.apache.kylin.metrics.lib.impl.hive.HiveSink",
"org.apache.kylin.metrics.lib.impl.hive.HiveSink"));
+ map.put("isConvertExpressionToCcEnabled",
+ new PropertiesEntity("kylin.query.implicit-computed-column-convert", "true", true));
+
+ map.put("isAggComputedColumnRewriteEnabled",
+ new PropertiesEntity("kylin.query.agg-computed-column-rewrite", "true", true));
+
+ map.put("getConvertCcMaxIterations",
+ new PropertiesEntity("kylin.query.computed-column-max-recursion-times", "10", 10));
map.put("isKylinMetricsMonitorEnabled", new PropertiesEntity("kylin.metrics.monitor-enabled", "false", false));
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/ComputedColumnRewriter.java b/src/query-common/src/main/java/org/apache/kylin/query/util/ComputedColumnRewriter.java
index 1e198f87db..ea4878190e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/ComputedColumnRewriter.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/ComputedColumnRewriter.java
@@ -24,9 +24,9 @@ import java.util.Map;
import org.apache.calcite.sql.SqlNode;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CollectionUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.ParameterDesc;
@@ -41,8 +41,6 @@ import org.apache.kylin.query.relnode.TableColRefWithRel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.var;
@@ -65,7 +63,7 @@ public class ComputedColumnRewriter {
private static void rewriteAggInnerCol(KylinConfig kylinConfig, OLAPContext context, NDataModel model,
QueryAliasMatchInfo matchInfo) {
- if (!KapConfig.getInstanceFromEnv().isAggComputedColumnRewriteEnabled()
+ if (!kylinConfig.isConvertExpressionToCcEnabled() || !kylinConfig.isAggComputedColumnRewriteEnabled()
|| CollectionUtils.isEmpty(model.getComputedColumnDescs())) {
return;
}
@@ -102,8 +100,9 @@ public class ComputedColumnRewriter {
return null;
}
+ boolean onlyReuseUserDefinedCC = kylinConfig.onlyReuseUserDefinedCC();
for (ComputedColumnDesc cc : model.getComputedColumnDescs()) {
- if (kylinConfig.isTableExclusionEnabled() && kylinConfig.onlyReuseUserDefinedCC() && cc.isAutoCC()) {
+ if (onlyReuseUserDefinedCC && cc.isAutoCC()) {
continue;
}
@@ -122,7 +121,7 @@ public class ComputedColumnRewriter {
private static void rewriteTopNInnerCol(KylinConfig kylinConfig, OLAPContext context, NDataModel model,
QueryAliasMatchInfo matchInfo) {
- if (CollectionUtils.isEmpty(model.getComputedColumnDescs()))
+ if (CollectionUtils.isEmpty(model.getComputedColumnDescs()) || !kylinConfig.isConvertExpressionToCcEnabled())
return;
context.getSortColumns().stream().filter(TblColRef::isInnerColumn).forEach(column -> {
@@ -146,7 +145,7 @@ public class ComputedColumnRewriter {
private static void rewriteGroupByInnerCol(KylinConfig kylinConfig, OLAPContext ctx, NDataModel model,
QueryAliasMatchInfo matchInfo) {
- if (CollectionUtils.isEmpty(model.getComputedColumnDescs())) {
+ if (CollectionUtils.isEmpty(model.getComputedColumnDescs()) || !kylinConfig.isConvertExpressionToCcEnabled()) {
return;
}
@@ -161,8 +160,10 @@ public class ComputedColumnRewriter {
tableColRefWIthRel.getTblColRef().getParserDescription(), e);
continue;
}
+
+ boolean onlyReuseUserDefinedCC = kylinConfig.onlyReuseUserDefinedCC();
for (ComputedColumnDesc cc : model.getComputedColumnDescs()) {
- if (kylinConfig.isTableExclusionEnabled() && kylinConfig.onlyReuseUserDefinedCC() && cc.isAutoCC()) {
+ if (onlyReuseUserDefinedCC && cc.isAutoCC()) {
continue;
}
SqlNode ccExpressionNode = CalciteParser.getExpNode(cc.getExpression());
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index ba1f57a832..1c4fa5a287 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -51,7 +51,6 @@ import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.calcite.util.Litmus;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.ThreadUtil;
@@ -70,7 +69,6 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.IQueryTransformer;
import lombok.SneakyThrows;
-import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -259,32 +257,22 @@ public class ConvertToComputedColumn implements IQueryTransformer {
return transformImpl(originSql, project, defaultSchema, dataModelDescs);
}
- private String transformImpl(String originSql, String project, String defaultSchema,
- List<NDataModel> dataModelDescs) throws SqlParseException {
-
+ private String transformImpl(String originSql, String project, String defaultSchema, List<NDataModel> models)
+ throws SqlParseException {
if (project == null || originSql == null) {
return originSql;
}
- return transformImpl(originSql, new QueryAliasMatcher(project, defaultSchema), dataModelDescs);
- }
-
- private String transformImpl(String originSql, QueryAliasMatcher queryAliasMatcher, List<NDataModel> dataModelDescs)
- throws SqlParseException {
- if (!KapConfig.getInstanceFromEnv().isImplicitComputedColumnConvertEnabled()) {
+ KylinConfig projectConfig = NProjectManager.getProjectConfig(project);
+ QueryAliasMatcher queryAliasMatcher = new QueryAliasMatcher(project, defaultSchema);
+ if (!projectConfig.isConvertExpressionToCcEnabled()) {
return originSql;
}
+ int maxRecursionTimes = projectConfig.getConvertCcMaxIterations();
String sql = originSql;
- if (queryAliasMatcher == null || sql == null) {
- return sql;
- }
-
- int recursionTimes = 0;
- int maxRecursionTimes = KapConfig.getInstanceFromEnv().getComputedColumnMaxRecursionTimes();
-
- while ((recursionTimes++) < maxRecursionTimes) {
- Pair<String, Boolean> result = transformImplRecursive(sql, queryAliasMatcher, dataModelDescs, false);
+ for (int i = 0; i < maxRecursionTimes; i++) {
+ Pair<String, Boolean> result = transformImplRecursive(sql, queryAliasMatcher, models);
sql = result.getFirst();
boolean recursionCompleted = result.getSecond();
if (recursionCompleted) {
@@ -296,7 +284,7 @@ public class ConvertToComputedColumn implements IQueryTransformer {
}
private Pair<String, Boolean> transformImplRecursive(String sql, QueryAliasMatcher queryAliasMatcher,
- List<NDataModel> dataModelDescs, boolean replaceCcName) throws SqlParseException {
+ List<NDataModel> models) throws SqlParseException {
boolean recursionCompleted = true;
List<SqlCall> selectOrOrderbys = SqlSubqueryFinder.getSubqueries(sql);
Pair<String, Integer> choiceForCurrentSubquery = null; //<new sql, number of changes by the model>
@@ -309,9 +297,9 @@ public class ConvertToComputedColumn implements IQueryTransformer {
SqlCall selectOrOrderby = selectOrOrderbys.get(i);
- ComputedColumnReplacer rewriteChecker = new ComputedColumnReplacer(queryAliasMatcher, dataModelDescs,
+ ComputedColumnReplacer rewriteChecker = new ComputedColumnReplacer(queryAliasMatcher, models,
recursionCompleted, choiceForCurrentSubquery, selectOrOrderby);
- rewriteChecker.replace(sql, replaceCcName);
+ rewriteChecker.replace(sql, false);
recursionCompleted = rewriteChecker.isRecursionCompleted();
choiceForCurrentSubquery = rewriteChecker.getChoiceForCurrentSubquery();
@@ -402,16 +390,9 @@ public class ConvertToComputedColumn implements IQueryTransformer {
Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, inputSql);
int start = startEndPos.getFirst();
int end = startEndPos.getSecond();
-
- boolean conflict = false;
- for (val pair : toBeReplacedExp) {
- Pair<Integer, Integer> replaced = pair.getSecond();
- if (!(replaced.getFirst() >= end || replaced.getSecond() <= start)) {
- // overlap with chosen areas
- conflict = true;
- break;
- }
- }
+ boolean conflict = toBeReplacedExp.stream().map(Pair::getSecond)
+ .anyMatch(replaced -> !(replaced.getFirst() >= end || replaced.getSecond() <= start));
+ // overlap with chosen areas
if (conflict) {
continue;
}
@@ -421,7 +402,8 @@ public class ConvertToComputedColumn implements IQueryTransformer {
return toBeReplacedExp;
}
- //Return matched node's position and its alias(if exists).If can not find matches, return an empty list
+ // Return matched node's position and its alias(if exists).
+ // If we can not find matches, return an empty list
private List<SqlNode> getMatchedNodes(SqlCall selectOrOrderby, String ccExp, QueryAliasMatchInfo matchInfo) {
if (ccExp == null || ccExp.equals(StringUtils.EMPTY)) {
return Collections.emptyList();
@@ -473,7 +455,7 @@ public class ConvertToComputedColumn implements IQueryTransformer {
}
static class SqlTreeVisitor implements SqlVisitor<SqlNode> {
- private List<SqlNode> sqlNodes;
+ private final List<SqlNode> sqlNodes;
SqlTreeVisitor() {
this.sqlNodes = new ArrayList<>();
@@ -537,11 +519,11 @@ public class ConvertToComputedColumn implements IQueryTransformer {
}
private class ComputedColumnReplacer {
- private QueryAliasMatcher queryAliasMatcher;
- private List<NDataModel> dataModels;
+ private final QueryAliasMatcher queryAliasMatcher;
+ private final List<NDataModel> dataModels;
private boolean recursionCompleted;
private Pair<String, Integer> choiceForCurrentSubquery;
- private SqlCall selectOrOrderby;
+ private final SqlCall selectOrOrderby;
ComputedColumnReplacer(QueryAliasMatcher queryAliasMatcher, List<NDataModel> dataModels,
boolean recursionCompleted, Pair<String, Integer> choiceForCurrentSubquery, SqlCall selectOrOrderby) {
@@ -597,7 +579,7 @@ public class ConvertToComputedColumn implements IQueryTransformer {
private List<ComputedColumnDesc> getSortedComputedColumnWithModel(NDataModel model) {
List<ComputedColumnDesc> ccList = model.getComputedColumnDescs();
KylinConfig projectConfig = NProjectManager.getProjectConfig(model.getProject());
- if (projectConfig.isTableExclusionEnabled() && projectConfig.onlyReuseUserDefinedCC()) {
+ if (projectConfig.onlyReuseUserDefinedCC()) {
ccList = ccList.stream().filter(cc -> !cc.isAutoCC()).collect(Collectors.toList());
}
return getCCListSortByLength(ccList);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java b/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
index 148f40fb6c..cfb58dcab7 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
@@ -35,23 +35,22 @@ import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.calcite.util.Litmus;
import org.apache.commons.collections.MapUtils;
-import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Iterables;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Iterables;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
//very similar to ConvertToComputedColumn in structure, maybe we should extract a common base class?
public class RestoreFromComputedColumn implements IPushDownConverter {
@@ -78,10 +77,11 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
QueryAliasMatcher queryAliasMatcher = new QueryAliasMatcher(project, defaultSchema);
int recursionTimes = 0;
- int maxRecursionTimes = KapConfig.getInstanceFromEnv().getComputedColumnMaxRecursionTimes();
+ int maxRecursionTimes = NProjectManager.getProjectConfig(project).getConvertCcMaxIterations();
while (recursionTimes < maxRecursionTimes) {
- QueryInterruptChecker.checkThreadInterrupted("Interrupted sql transformation at the stage of RestoreFromComputedColumn",
+ QueryInterruptChecker.checkThreadInterrupted(
+ "Interrupted sql transformation at the stage of RestoreFromComputedColumn",
"Current step: SQL transformation");
recursionTimes++;
boolean recursionCompleted = true;
@@ -181,7 +181,8 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
for (NDataModel model : modelMap.values()) {
QueryAliasMatchInfo info = model.getComputedColumnDescs().isEmpty() ? null
: queryAliasMatcher.match(model, sqlSelect);
- QueryInterruptChecker.checkThreadInterrupted("Interrupted sql transformation at the stage of RestoreFromComputedColumn",
+ QueryInterruptChecker.checkThreadInterrupted(
+ "Interrupted sql transformation at the stage of RestoreFromComputedColumn",
"Current step: SQL transformation");
if (info == null) {
continue;
(kylin) 34/35: KYLIN-5845 Avoid model initialization for multi-thread
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0bbb74ee794beb17a8a501f5df5dfcc421bfd2a1
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Tue Oct 31 11:11:03 2023 +0800
KYLIN-5845 Avoid model initialization for multi-thread
---
.../main/java/org/apache/kylin/common/util/AddressUtil.java | 4 ++++
.../java/org/apache/kylin/common/util/AddressUtilTest.java | 1 +
.../org/apache/kylin/metadata/model/ColExcludedChecker.java | 13 +++++++------
.../java/org/apache/kylin/metadata/model/NDataModel.java | 4 ----
.../org/apache/kylin/metadata/model/NDataModelManager.java | 4 ----
.../java/org/apache/kylin/tool/MetadataToolTestFixture.java | 2 +-
6 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
index 71cd033143..bbb80a332f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
@@ -107,4 +107,8 @@ public class AddressUtil {
public static String concatInstanceName() {
return AddressUtil.getLocalHostExactAddress() + ":" + KylinConfig.getInstanceFromEnv().getServerPort();
}
+
+ public static void clearLocalIpAddressCache() {
+ localIpAddressCache = null;
+ }
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java b/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
index 5277bec0c6..0112d7b2ba 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
@@ -72,6 +72,7 @@ public class AddressUtilTest {
val old = getTestConfig().getServerIpAddress();
val mockIp = "192.168.1.101";
getTestConfig().setProperty("kylin.env.ip-address", mockIp);
+ AddressUtil.clearLocalIpAddressCache();
val servIp = AddressUtil.getLocalHostExactAddress();
Assert.assertEquals(servIp, mockIp);
if (!StringUtils.isEmpty(old)) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
index 3f63975bc4..ab2ae1e877 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
@@ -29,12 +29,11 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
-
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -84,11 +83,13 @@ public class ColExcludedChecker {
collectExcludedComputedColumns(config, project, model);
}
- private void collectExcludedComputedColumns(KylinConfig config, String project, NDataModel model) {
- if (model == null || model.isBroken()) {
+ private void collectExcludedComputedColumns(KylinConfig config, String project, NDataModel originModel) {
+ if (originModel == null || originModel.isBroken()) {
return;
}
- if (!model.isInitAlready()) {
+ NDataModel model = originModel;
+ if (originModel.getAllTables().isEmpty() || originModel.getEffectiveCols() == null) {
+ model = NDataModelManager.getInstance(config, project).copyForWrite(originModel);
model.init(config, project, Lists.newArrayList());
}
model.getAllTables().stream().filter(Objects::nonNull) //
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 183b5ebc59..3e981ade03 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -286,9 +286,6 @@ public class NDataModel extends RootPersistentEntity {
// mark this model as used for model save checking
private boolean saveCheck = false;
- // mark this model has invoked init() function
- private boolean initAlready = false;
-
public enum ColumnStatus {
TOMB, EXIST, DIMENSION
}
@@ -1040,7 +1037,6 @@ public class NDataModel extends RootPersistentEntity {
throw new KylinException(TABLE_JOIN_RELATIONSHIP_ERROR,
MsgPicker.getMsg().getDimensionTableUsedInThisModel());
}
- this.setInitAlready(true);
}
public ComputedColumnUtil.CCConflictInfo checkCCFailAtEnd(KylinConfig config, String project,
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
index 0fe43411d8..0e11b35ade 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
@@ -331,10 +331,6 @@ public class NDataModelManager {
return crud.copyForWrite(nDataModel);
}
- public NDataModel copy(NDataModel nDataModel) {
- return copyBySerialization(nDataModel);
- }
-
/**
* copyBySerialization will not use cache
*/
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTestFixture.java b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTestFixture.java
index db34af1149..71274e0535 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTestFixture.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTestFixture.java
@@ -48,7 +48,7 @@ public class MetadataToolTestFixture {
val dataModel2 = dataModelMgr.getDataModelDescByAlias("nmodel_basic_inner");
dataModelMgr.dropModel(dataModel2);
- val dataModel3 = dataModelMgr.copy(dataModel2);
+ val dataModel3 = dataModelMgr.copyBySerialization(dataModel2);
dataModel3.setUuid(RandomUtil.randomUUIDStr());
dataModel3.setAlias("data_model_3");
dataModel3.setMvcc(-1L);
(kylin) 10/35: KYLIN-5802 Models with the same name exist when restore metadata
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9b8050e00af23094104bf35439a5fd36470aa23a
Author: Hang Jia <75...@qq.com>
AuthorDate: Thu Aug 31 11:13:06 2023 +0800
KYLIN-5802 Models with the same name exist when restore metadata
---
.../kylin/common/exception/code/ErrorCodeTool.java | 3 +-
.../resources/kylin_error_msg_conf_cn.properties | 1 +
.../resources/kylin_error_msg_conf_en.properties | 1 +
.../main/resources/kylin_errorcode_conf.properties | 1 +
.../apache/kylin/helper/MetadataToolHelper.java | 118 +++++++++++++++++++++
.../org/apache/kylin/tool/MetadataToolTest.java | 57 ++++++++++
6 files changed, 180 insertions(+), 1 deletion(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
index 61cf7d13cd..6a7a46bee4 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
@@ -24,7 +24,8 @@ public enum ErrorCodeTool implements ErrorCodeProducer {
"KE-050040203"), PARAMETER_TIMESTAMP_COMPARE("KE-050040204"),
// path & file
- PATH_NOT_EXISTS("KE-050041201"), FILE_ALREADY_EXISTS("KE-050041202");
+ PATH_NOT_EXISTS("KE-050041201"), FILE_ALREADY_EXISTS("KE-050041202"),
+ MODEL_DUPLICATE_UUID_FAILED("KE-050041203");
private final ErrorCode errorCode;
private final ErrorMsg errorMsg;
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 010c2b10c3..2e19879659 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -226,6 +226,7 @@ KE-050040204=参数 “-endTime” <= 参数 “-startTime” 。
## 500412XX
KE-050041201=该路径不存在：%s。
KE-050041202=该路径已存在：%s。
+KE-050041203=请修改模型名称后再恢复:[project]:models: %s
# Common
## KE-060100201
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 75f05b384b..f9d66504ca 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -226,6 +226,7 @@ KE-050040204=Parameter "-endTime" <= Parameter "-startTime".
## 500412XX
KE-050041201=The path does not exist: %s.
KE-050041202=The path already exists: %s.
+KE-050041203=Please modify the model name and then restore:[project]:models: %s
# Common
## KE-060100201
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index f8ed420188..f8dd010445 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -237,6 +237,7 @@ KE-050040204
## 500412XX
KE-050041201
KE-050041202
+KE-050041203
# Common
KE-060100201
diff --git a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index f000e766a6..f3d920d918 100644
--- a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
+++ b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -19,6 +19,7 @@
package org.apache.kylin.helper;
import static org.apache.kylin.common.exception.code.ErrorCodeTool.FILE_ALREADY_EXISTS;
+import static org.apache.kylin.common.exception.code.ErrorCodeTool.MODEL_DUPLICATE_UUID_FAILED;
import java.io.File;
import java.io.IOException;
@@ -28,9 +29,12 @@ import java.nio.file.Paths;
import java.time.Clock;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.NavigableSet;
import java.util.Objects;
import java.util.Set;
@@ -58,8 +62,10 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.tool.HDFSMetadataTool;
import org.apache.kylin.tool.constant.StringConstant;
@@ -288,6 +294,7 @@ public class MetadataToolHelper {
public void restore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore, String project,
boolean delete) {
+ checkDuplicateUuidModel(currentResourceStore, restoreResourceStore, project, delete);
if (StringUtils.isBlank(project)) {
logger.info("start to restore all projects");
var srcProjectFolders = restoreResourceStore.listResources("/");
@@ -340,6 +347,117 @@ public class MetadataToolHelper {
logger.info("restore successfully");
}
+ public void checkDuplicateUuidModel(ResourceStore currentResourceStore, ResourceStore restoreResourceStore,
+ String project, boolean delete) {
+ logger.info("start check duplicate uuid model");
+ if (delete) {
+ return;
+ }
+
+ Map<String, List<String>> duplicateUuidModelByProject = getDuplicateUuidModelByProject(currentResourceStore,
+ restoreResourceStore, project);
+
+ if (!duplicateUuidModelByProject.isEmpty()) {
+ String errorMsg = duplicateUuidModelByProject.entrySet().stream()
+ .map(m -> "[" + m.getKey() + "]:" + String.join(",", m.getValue()))
+ .collect(Collectors.joining(";"));
+ String info = String.format(
+ "[UNEXPECTED_THINGS_HAPPENED] There will be models with the same name after recovery, please rename these models first:[project]:models: %s ",
+ errorMsg);
+ logger.error(info);
+ System.out.println(StringConstant.ANSI_RED + info + StringConstant.ANSI_RESET);
+ throw new KylinException(MODEL_DUPLICATE_UUID_FAILED, errorMsg);
+ }
+ logger.info("end check duplicate uuid model");
+ }
+
+ private Map<String, List<String>> getDuplicateUuidModelByProject(ResourceStore currentResourceStore,
+ ResourceStore restoreResourceStore, String project) {
+ Map<String, List<String>> duplicateUuidModelByProject = Maps.newHashMap();
+ if (StringUtils.isBlank(project)) {
+ duplicateUuidModelByProject = getDuplicateUuidModelByAllProject(currentResourceStore, restoreResourceStore);
+ } else {
+ List<String> duplicateUuidModel = getDuplicateUuidModel(currentResourceStore, restoreResourceStore,
+ project);
+ if (!duplicateUuidModel.isEmpty()) {
+ duplicateUuidModelByProject.put(project, duplicateUuidModel);
+ }
+ }
+ return duplicateUuidModelByProject;
+ }
+
+ private Map<String, List<String>> getDuplicateUuidModelByAllProject(ResourceStore currentResourceStore,
+ ResourceStore restoreResourceStore) {
+ var destProjectFolders = currentResourceStore.listResources("/");
+ var srcProjectFolders = restoreResourceStore.listResources("/");
+ destProjectFolders = destProjectFolders == null ? Sets.newTreeSet() : destProjectFolders;
+ srcProjectFolders = srcProjectFolders == null ? Sets.newTreeSet() : srcProjectFolders;
+ val projectFolders = Sets.union(srcProjectFolders, destProjectFolders);
+
+ Map<String, List<String>> duplicateUuidModelByProject = Maps.newHashMap();
+ for (String projectPath : projectFolders) {
+ if (projectPath.equals(ResourceStore.METASTORE_UUID_TAG)
+ || projectPath.equals(ResourceStore.METASTORE_IMAGE)) {
+ continue;
+ }
+ val projectName = Paths.get(projectPath).getName(0).toString();
+ List<String> duplicateUuidModel = getDuplicateUuidModel(currentResourceStore, restoreResourceStore,
+ projectName);
+ if (!duplicateUuidModel.isEmpty()) {
+ duplicateUuidModelByProject.put(projectName, duplicateUuidModel);
+ }
+ }
+ return duplicateUuidModelByProject;
+ }
+
+ private List<String> getDuplicateUuidModel(ResourceStore currentResourceStore, ResourceStore restoreResourceStore,
+ String projectName) {
+ String modelDescRootPath = File.separator + projectName + ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT;
+ Set<String> destModelResource = currentResourceStore.listResources(modelDescRootPath);
+ Set<String> srcModelResource = restoreResourceStore.listResources(modelDescRootPath);
+ destModelResource = destModelResource == null ? Collections.emptySet() : destModelResource;
+ srcModelResource = srcModelResource == null ? Collections.emptySet() : srcModelResource;
+
+ Sets.SetView<String> insertsModelResource = Sets.difference(srcModelResource, destModelResource);
+
+ List<NDataModel> allModels = new ArrayList<>(
+ getModelListFromResource(projectName, destModelResource, currentResourceStore));
+ List<NDataModel> insertsModels = getModelListFromResource(projectName, new HashSet<>(insertsModelResource),
+ restoreResourceStore);
+ allModels.addAll(insertsModels);
+
+ Map<String, Set<String>> nameUuids = Maps.newHashMap();
+ for (NDataModel model : allModels) {
+ String modelAlias = model.getAlias();
+ nameUuids.putIfAbsent(modelAlias, Sets.newHashSet());
+ nameUuids.get(modelAlias).add(model.getUuid());
+ }
+ return nameUuids.entrySet().stream().filter(m -> m.getValue().size() > 1).map(Map.Entry::getKey)
+ .collect(Collectors.toList());
+ }
+
+ public List<NDataModel> getModelListFromResource(String projectName, Set<String> modelResource,
+ ResourceStore resourceStore) {
+
+ if (modelResource == null) {
+ return new ArrayList<>();
+ }
+ List<NDataModel> models = new ArrayList<>();
+ for (String resource : modelResource) {
+ try {
+ NDataModel nDataModel = JsonUtil.readValue(resourceStore.getResource(resource).getByteSource().read(),
+ NDataModel.class);
+ nDataModel.setProject(projectName);
+ models.add(nDataModel);
+ } catch (IOException e) {
+ if (!KylinConfig.getInstanceFromEnv().isUTEnv()) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+ return models;
+ }
+
private int doRestore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore,
Set<String> destResources, Set<String> srcResources, boolean delete) throws IOException {
val threadViewRS = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
index b90139a94e..3bb9652edb 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.tool;
import static org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil.datasourceParameters;
+import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -64,6 +65,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.OptionBuilder;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
@@ -452,6 +454,61 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
}
+ @Test
+ public void testRestoreDuplicateUuidModel() throws Exception {
+ val restoreFolder = temporaryFolder.newFolder();
+ MetadataToolTestFixture.fixtureRestoreTest(getTestConfig(), restoreFolder, "/");
+
+ val tool = new MetadataTool(getTestConfig());
+ tool.execute(new String[] { "-restore", "-dir", restoreFolder.getAbsolutePath() });
+ NDataModelManager dataModelManager = NDataModelManager.getInstance(getTestConfig(), "default");
+
+ String modelId = "82fa7671-a935-45f5-8779-85703601f49a";
+ NDataModel dataModelDesc = dataModelManager.getDataModelDesc(modelId);
+ dataModelManager.dropModel(dataModelDesc.getUuid());
+ NDataModel nDataModel = dataModelManager.copyBySerialization(dataModelDesc);
+ nDataModel.setMvcc(-1);
+ nDataModel.setUuid(RandomUtil.randomUUIDStr());
+ dataModelManager.createDataModelDesc(nDataModel, nDataModel.getOwner());
+
+ String modelId2 = "a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94";
+ NDataModel dataModelDesc2 = dataModelManager.getDataModelDesc(modelId2);
+ dataModelManager.dropModel(dataModelDesc2.getUuid());
+ NDataModel nDataModel2 = dataModelManager.copyBySerialization(dataModelDesc2);
+ nDataModel2.setMvcc(-1);
+ nDataModel2.setUuid(RandomUtil.randomUUIDStr());
+ dataModelManager.createDataModelDesc(nDataModel2, nDataModel2.getOwner());
+
+ try {
+ tool.execute(new String[] { "-restore", "-dir", restoreFolder.getAbsolutePath() });
+ } catch (Exception e) {
+ assertTrue(e instanceof KylinException);
+ assertEquals(
+ "KE-050041203: Please modify the model name and then restore:[project]:models: [default]:ut_inner_join_cube_partial,test_encoding",
+ e.getCause().toString());
+ }
+ long start = System.currentTimeMillis() / 1000;
+ await().until(() -> start != (System.currentTimeMillis() / 1000));
+ try {
+ tool.execute(new String[] { "-restore", "-project", "default", "-dir", restoreFolder.getAbsolutePath() });
+ } catch (Exception e) {
+ assertTrue(e instanceof KylinException);
+ assertEquals(
+ "KE-050041203: Please modify the model name and then restore:[project]:models: [default]:ut_inner_join_cube_partial,test_encoding",
+ e.getCause().toString());
+ }
+
+ Assertions.assertThat(dataModelManager.getDataModelDesc(modelId)).isNull();
+ Assertions.assertThat(dataModelManager.getDataModelDesc(modelId2)).isNull();
+
+ long start2 = System.currentTimeMillis() / 1000;
+ await().until(() -> start2 != (System.currentTimeMillis() / 1000));
+ tool.execute(new String[] { "-restore", "-dir", restoreFolder.getAbsolutePath(), "--after-truncate" });
+
+ Assertions.assertThat(dataModelManager.getDataModelDesc(modelId)).isNotNull();
+ Assertions.assertThat(dataModelManager.getDataModelDesc(modelId2)).isNotNull();
+ }
+
@Test
public void testRestoreOverwriteAllCompressWithSrcOrDestIsEmpty() throws Exception {
val emptyFolder = temporaryFolder.newFolder();
(kylin) 30/35: KYLIN-5839 Block agg-push-down for non-equal-join
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit be22ca0de7e512df5dc443b99ed6df63635f9dfe
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Mon Oct 2 17:48:57 2023 +0800
KYLIN-5839 Block agg-push-down for non-equal-join
KYLIN-5382 only deals with the equal-join conditions and gives the
corresponding test cases.
KYLIN-5831 fixed the problem where non-equiv-join conditions were
incorrectly converted to equivalent conditions. Due to the lack of test
cases, a regression bug occurs in ContextUtil.isJoinFromSameContext.
The related non-equiv-join sql is as follows.
If you want a corresponding equal-join sql, just remove the condition
content `or (t2.fcol_10 is null and t5.fcol_17 is null)`.
```sql
select t2.fcol_7 fcol_7, count(distinct t2.fcol_6) fcol_6
from (select t1.company_code fcol_6, t1.type_name fcol_7,
case when t1.created_date = t0.fcol_1 then 'TRUE'
else 'FALSE' end fcol_10
from ( select company_code, created_date, type_name
from "DEFAULT"."TEST_AGG_PUSH_DOWN" ) t1
join ( select company_code, max(created_date) fcol_1
from "DEFAULT"."TEST_AGG_PUSH_DOWN"
group by company_code
) t0 on t1.company_code = t0.company_code
) t2 join ( select 'TRUE' fcol_17 ) t5
on t2.fcol_10 = t5.fcol_17 or (t2.fcol_10 is null and t5.fcol_17 is null)
group by t2.fcol_7
```
---
.../apache/kylin/query/relnode/ContextUtil.java | 24 ++++++++++++++--------
.../kylin/query/relnode/KapNonEquiJoinRel.java | 3 +++
2 files changed, 19 insertions(+), 8 deletions(-)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
index d08d0d073d..4a063cf82f 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
@@ -36,15 +36,14 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexSlot;
import org.apache.calcite.sql.fun.SqlCountAggFunction;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.util.RexUtils;
import org.apache.kylin.util.CalciteSystemProperty;
import org.slf4j.Logger;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
public class ContextUtil {
private ContextUtil() {
}
@@ -203,12 +202,21 @@ public class ContextUtil {
return false;
}
if (potentialSubRel instanceof KapProjectRel) {
- ((KapJoinRel) joinRel).leftKeys.forEach(leftKey -> {
- RexNode leftCol = ((KapProjectRel) potentialSubRel).getProjects().get(leftKey);
- if (leftCol instanceof RexCall) {
- indexOfInputCols.add(leftKey);
+ if (joinRel instanceof KapJoinRel) {
+ ((KapJoinRel) joinRel).leftKeys.forEach(leftKey -> {
+ RexNode leftCol = ((KapProjectRel) potentialSubRel).getProjects().get(leftKey);
+ if (leftCol instanceof RexCall) {
+ indexOfInputCols.add(leftKey);
+ }
+ });
+ } else {
+ // non-equiv-join rel: treat the left and right subtrees as a context,
+ // and refuse to push agg down.
+ KapNonEquiJoinRel nonEquivJoinRel = (KapNonEquiJoinRel) joinRel;
+ if (!nonEquivJoinRel.isScd2Rel()) {
+ return false;
}
- });
+ }
}
return derivedFromSameContext(indexOfInputCols, potentialSubRel, subContext, hasCountConstant);
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
index d3cfda0c2c..0b9840cc1b 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
@@ -63,6 +63,8 @@ import org.apache.kylin.query.util.ICutContextStrategy;
import org.apache.kylin.query.util.RexToTblColRefTranslator;
import org.apache.kylin.query.util.RexUtils;
+import lombok.Getter;
+
public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
private OLAPContext context;
@@ -75,6 +77,7 @@ public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
// record left input size before rewrite for runtime join expression parsing
private int leftInputSizeBeforeRewrite = -1;
+ @Getter
private final boolean isScd2Rel;
public KapNonEquiJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition,
(kylin) 14/35: KYLIN-5808 Optimize performance when saving model/adding CC
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c4aad6e78b24c36cad6ce18a2d00dfd86527be0d
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Sat Sep 2 09:16:38 2023 +0800
KYLIN-5808 Optimize performance when saving model/adding CC
---
.../engine/spark/utils/ComputedColumnEvalUtil.java | 20 +++-----
.../job/stage/build/FlatTableAndDictBase.scala | 57 ++++++++++++++++++++
.../spark/smarter/IndexDependencyParser.scala | 60 +++++++++++++++-------
3 files changed, 106 insertions(+), 31 deletions(-)
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
index 603dbd7a5a..f7e9b7936e 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.engine.spark.job.NSparkCubingUtil;
import org.apache.kylin.engine.spark.smarter.IndexDependencyParser;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.model.BadModelException;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
@@ -33,13 +34,9 @@ import org.apache.kylin.metadata.model.exception.IllegalCCExpressionException;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SparderEnv;
-import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.util.SparderTypeUtil;
import org.springframework.util.CollectionUtils;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -62,7 +59,7 @@ public class ComputedColumnEvalUtil {
public static void evalDataTypeOfCCInAuto(List<ComputedColumnDesc> computedColumns, NDataModel nDataModel,
int start, int end) {
try {
- evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(), nDataModel, start, end);
+ evalDataTypeOfCC(computedColumns, nDataModel, start, end);
} catch (Exception e) {
if (end - start > 1) { //numbers of CC > 1
evalDataTypeOfCCInAuto(computedColumns, nDataModel, start, start + (end - start) / 2);
@@ -80,7 +77,7 @@ public class ComputedColumnEvalUtil {
return;
}
try {
- evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(), nDataModel, 0, computedColumns.size());
+ evalDataTypeOfCC(computedColumns, nDataModel, 0, computedColumns.size());
} catch (Exception e) {
evalDataTypeOfCCInManual(computedColumns, nDataModel, 0, computedColumns.size());
}
@@ -90,7 +87,7 @@ public class ComputedColumnEvalUtil {
int start, int end) {
for (int i = start; i < end; i++) {
try {
- evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(), nDataModel, i, i + 1);
+ evalDataTypeOfCC(computedColumns, nDataModel, i, i + 1);
} catch (Exception e) {
Preconditions.checkNotNull(computedColumns.get(i));
throw new IllegalCCExpressionException(QueryErrorCode.CC_EXPRESSION_ILLEGAL,
@@ -101,15 +98,14 @@ public class ComputedColumnEvalUtil {
}
}
- private static void evalDataTypeOfCC(List<ComputedColumnDesc> computedColumns, SparkSession ss,
- NDataModel nDataModel, int start, int end) {
+ private static void evalDataTypeOfCC(List<ComputedColumnDesc> computedColumns, NDataModel nDataModel, int start,
+ int end) {
IndexDependencyParser parser = new IndexDependencyParser(nDataModel);
- Dataset<Row> originDf = parser.generateFullFlatTableDF(ss, nDataModel);
- originDf.persist();
+ Dataset<Row> df = parser.getFullFlatTableDataFrame(nDataModel);
String[] ccExprArray = computedColumns.subList(start, end).stream() //
.map(ComputedColumnDesc::getInnerExpression) //
.map(NSparkCubingUtil::convertFromDotWithBackTick).toArray(String[]::new);
- Dataset<Row> ds = originDf.selectExpr(ccExprArray);
+ Dataset<Row> ds = df.selectExpr(ccExprArray);
for (int i = start; i < end; i++) {
String dataType = SparderTypeUtil.convertSparkTypeToSqlType(ds.schema().fields()[i - start].dataType());
computedColumns.get(i).setDatatype(dataType);
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 4634b77096..935eaefaea 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -46,6 +46,8 @@ import org.apache.kylin.metadata.cube.planner.CostBasePlannerUtils
import org.apache.kylin.metadata.model._
import org.apache.spark.sql.KapFunctions.dict_encode_v3
import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.plans.JoinType
+import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint, LogicalPlan}
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.SparderTypeUtil
@@ -718,6 +720,19 @@ object FlatTableAndDictBase extends LogEx {
newDS
}
+ def wrapAlias(originPlan: LogicalPlan, alias: String, needLog: Boolean): LogicalPlan = {
+ val newFields = originPlan.output
+ .map(f => {
+ val aliasDotColName = "`" + alias + "`" + "." + "`" + f.name + "`"
+ convertFromDot(aliasDotColName)
+ })
+ val newDS = SparkOperation.projectAsAlias(newFields, originPlan)
+ if (needLog) {
+ logInfo(s"Wrap ALIAS ${originPlan.schema.treeString} TO ${newDS.schema.treeString}")
+ }
+ newDS
+ }
+
def joinFactTableWithLookupTables(rootFactDataset: Dataset[Row],
lookupTableDatasetMap: mutable.Map[JoinTableDesc, Dataset[Row]],
@@ -728,6 +743,15 @@ object FlatTableAndDictBase extends LogEx {
joinTableDataset(model.getRootFactTable.getTableDesc, tuple._1, joinedDataset, tuple._2, needLog))
}
+ def joinFactTableWithLookupTables(rootFactPlan: LogicalPlan,
+ lookupTableDatasetMap: mutable.Map[JoinTableDesc, LogicalPlan],
+ model: NDataModel,
+ needLog: Boolean): LogicalPlan = {
+ lookupTableDatasetMap.foldLeft(rootFactPlan)(
+ (joinedDataset: LogicalPlan, tuple: (JoinTableDesc, LogicalPlan)) =>
+ joinTableLogicalPlan(model.getRootFactTable.getTableDesc, tuple._1, joinedDataset, tuple._2, needLog))
+ }
+
def joinTableDataset(rootFactDesc: TableDesc,
lookupDesc: JoinTableDesc,
rootFactDataset: Dataset[Row],
@@ -765,6 +789,39 @@ object FlatTableAndDictBase extends LogEx {
afterJoin
}
+ def joinTableLogicalPlan(rootFactDesc: TableDesc,
+ lookupDesc: JoinTableDesc,
+ rootFactPlan: LogicalPlan,
+ lookupPlan: LogicalPlan,
+ needLog: Boolean = true): LogicalPlan = {
+ var afterJoin = rootFactPlan
+ val join = lookupDesc.getJoin
+ if (join != null && !StringUtils.isEmpty(join.getType)) {
+ val joinType = join.getType.toUpperCase(Locale.ROOT)
+ val pk = join.getPrimaryKeyColumns
+ val fk = join.getForeignKeyColumns
+ if (pk.length != fk.length) {
+ throw new RuntimeException(
+ s"Invalid join condition of fact table: $rootFactDesc,fk: ${fk.mkString(",")}," +
+ s" lookup table:$lookupDesc, pk: ${pk.mkString(",")}")
+ }
+ val equiConditionColPairs = fk.zip(pk).map(joinKey =>
+ col(convertFromDot(joinKey._1.getBackTickIdentity))
+ .equalTo(col(convertFromDot(joinKey._2.getBackTickIdentity))))
+
+ if (join.getNonEquiJoinCondition != null) {
+ val condition: Column = getCondition(join)
+ logInfo(s"Root table ${rootFactDesc.getIdentity}, join table ${lookupDesc.getAlias}, non-equi condition: ${condition.toString()}")
+ afterJoin = Join(afterJoin, lookupPlan, JoinType.apply(joinType), Option.apply(condition.expr), JoinHint.NONE)
+ } else {
+ val condition = equiConditionColPairs.reduce(_ && _)
+ logInfo(s"Root table ${rootFactDesc.getIdentity}, join table ${lookupDesc.getAlias}, condition: ${condition.toString()}")
+ afterJoin = Join(afterJoin, lookupPlan, JoinType.apply(joinType), Option.apply(condition.expr), JoinHint.NONE)
+ }
+ }
+ afterJoin
+ }
+
def getCondition(join: JoinDesc): Column = {
val pk = join.getPrimaryKeyColumns
val fk = join.getForeignKeyColumns
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
index 44e52099d7..9da76b7996 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/smarter/IndexDependencyParser.scala
@@ -17,9 +17,6 @@
*/
package org.apache.kylin.engine.spark.smarter
-import java.util
-import java.util.Collections
-
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.engine.spark.job.NSparkCubingUtil
@@ -28,10 +25,15 @@ import org.apache.kylin.guava30.shaded.common.collect.{Lists, Maps, Sets}
import org.apache.kylin.metadata.cube.model.LayoutEntity
import org.apache.kylin.metadata.model._
import org.apache.kylin.query.util.PushDownUtil
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, SubqueryAlias}
import org.apache.spark.sql.execution.utils.SchemaProcessor
-import org.apache.spark.sql.types.StructField
-import org.apache.spark.sql.{Dataset, Row, SparderEnv, SparkSession}
+import org.apache.spark.sql.functions.col
+import org.apache.spark.sql.types.{StructField, StructType}
+import java.util
+import java.util.Collections
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -40,8 +42,15 @@ class IndexDependencyParser(val model: NDataModel) {
private val ccTableNameAliasMap = Maps.newHashMap[String, util.Set[String]]
private val joinTableAliasMap = Maps.newHashMap[String, util.Set[String]]
private val allTablesAlias = Sets.newHashSet[String]
+ private var fullFlatTableDF : Option[Dataset[Row]] = None
initTableNames()
-
+ def getFullFlatTableDataFrame(model: NDataModel): Dataset[Row] = {
+ if (fullFlatTableDF.isDefined) {
+ fullFlatTableDF.get
+ } else {
+ generateFullFlatTableDF(model)
+ }
+ }
def getRelatedTablesAlias(layouts: util.Collection[LayoutEntity]): util.List[String] = {
val relatedTables = Sets.newHashSet[String]
layouts.asScala.foreach(layout => relatedTables.addAll(getRelatedTablesAlias(layout)))
@@ -105,34 +114,47 @@ class IndexDependencyParser(val model: NDataModel) {
}
}
- def generateFullFlatTableDF(ss: SparkSession, model: NDataModel): Dataset[Row] = {
- val rootDF = generateDatasetOnTable(ss, model.getRootFactTable)
+ def generateFullFlatTableDF(model: NDataModel): Dataset[Row] = {
+ val rootLogicalPlan = generateLogicalPlanOnTable(model.getRootFactTable)
// look up tables
- val joinTableDFMap = mutable.LinkedHashMap[JoinTableDesc, Dataset[Row]]()
+ val joinTableDFMap = mutable.LinkedHashMap[JoinTableDesc, LogicalPlan]()
model.getJoinTables.asScala.map((joinTable: JoinTableDesc) => {
- joinTableDFMap.put(joinTable, generateDatasetOnTable(ss, joinTable.getTableRef))
+ joinTableDFMap.put(joinTable, generateLogicalPlanOnTable(joinTable.getTableRef))
})
- val df = FlatTableAndDictBase.joinFactTableWithLookupTables(rootDF, joinTableDFMap, model, needLog = false)
+ val df = FlatTableAndDictBase.joinFactTableWithLookupTables(rootLogicalPlan, joinTableDFMap, model, needLog = false)
val filterCondition = model.getFilterCondition
if (StringUtils.isNotEmpty(filterCondition)) {
val massagedCondition = PushDownUtil.massageExpression(model, model.getProject, filterCondition, null)
- df.where(NSparkCubingUtil.convertFromDotWithBackTick(massagedCondition))
+ val condition = NSparkCubingUtil.convertFromDot(massagedCondition)
+ SparkOperation.filter(col(condition), df)
}
- df
+ SparkInternalAgent.getDataFrame(SparderEnv.getSparkSession, df)
}
- private def generateDatasetOnTable(ss: SparkSession, tableRef: TableRef): Dataset[Row] = {
+ private def generateLogicalPlanOnTable(tableRef: TableRef): LogicalPlan = {
val tableCols = tableRef.getColumns.asScala.map(_.getColumnDesc).toArray
val structType = SchemaProcessor.buildSchemaWithRawTable(tableCols)
val alias = tableRef.getAlias
- val dataset = ss.createDataFrame(Lists.newArrayList[Row], structType).alias(alias)
- FlatTableAndDictBase.wrapAlias(dataset, alias, needLog = false)
+ val fsRelation = LocalRelation(toAttributes(structType))
+ val plan = SubqueryAlias(alias, fsRelation)
+ FlatTableAndDictBase.wrapAlias(plan, alias, needLog = false)
+ }
+
+ def toAttribute(field: StructField): AttributeReference =
+ AttributeReference(field.name, field.dataType, field.nullable, field.metadata)()
+
+ /**
+ * Convert a [[StructType]] into a Seq of [[AttributeReference]].
+ */
+ def toAttributes(schema: StructType): Seq[AttributeReference] = {
+ schema.map(toAttribute)
}
private def initTableNames(): Unit = {
- val ccList = model.getComputedColumnDescs
- val originDf = generateFullFlatTableDF(SparderEnv.getSparkSession, model)
+ fullFlatTableDF = Option(generateFullFlatTableDF(model))
+ val originDf = fullFlatTableDF.get
val colFields = originDf.schema.fields
+ val ccList = model.getComputedColumnDescs
val ds = originDf.selectExpr(ccList.asScala.map(_.getInnerExpression)
.map(NSparkCubingUtil.convertFromDotWithBackTick): _*)
ccList.asScala.zip(ds.schema.fields).foreach(pair => {
@@ -156,7 +178,7 @@ class IndexDependencyParser(val model: NDataModel) {
def unwrapComputeColumn(ccInnerExpression: String): java.util.Set[TblColRef] = {
val result: util.Set[TblColRef] = Sets.newHashSet()
- val originDf = generateFullFlatTableDF(SparderEnv.getSparkSession, model)
+ val originDf = getFullFlatTableDataFrame(model)
val colFields = originDf.schema.fields
val ccDs = originDf.selectExpr(NSparkCubingUtil.convertFromDotWithBackTick(ccInnerExpression))
ccDs.schema.fields.foreach(fieldName => {
(kylin) 24/35: KYLIN-5830 Push columns of sortRel to subOlapContexts
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 191d08c272b48d04441783290459291f70c2e9e6
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Wed Sep 20 10:20:56 2023 +0800
KYLIN-5830 Push columns of sortRel to subOlapContexts
---
.../query/routing/RealizationChooserTest.java | 24 +++++++++++++++++++++-
.../org/apache/kylin/util/OlapContextTestUtil.java | 13 ++++++++++++
.../org/apache/kylin/query/relnode/KapSortRel.java | 19 +++++++++++++++--
.../apache/kylin/query/relnode/KapTableScan.java | 10 +++++----
4 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
index 2cea9cf51b..267be6b1d8 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
@@ -21,12 +21,14 @@ package org.apache.kylin.query.routing;
import java.util.List;
import java.util.Map;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.query.relnode.KapSortRel;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.After;
@@ -97,6 +99,25 @@ public class RealizationChooserTest extends NLocalWithSparkSessionTest {
}
}
+ @Test
+ public void testPushSortRelToSubOlapContexts() throws SqlParseException {
+ overwriteSystemProp("kylin.query.print-logical-plan", "true");
+ String project = "joins_graph_left_or_inner";
+ String sql = "select a.NAME from TEST_BANK_INCOME a inner join TEST_BANK_LOCATION b on a.COUNTRY = b.COUNTRY\n"
+ + "order by a.INCOME nulls last";
+ RelNode relNode = OlapContextTestUtil.cutOlapContextsAndReturnRelNode(project, sql);
+ KapSortRel sortRel = null;
+ while (relNode != null) {
+ if (relNode instanceof KapSortRel) {
+ sortRel = (KapSortRel) relNode;
+ break;
+ }
+ relNode = relNode.getInput(0);
+ }
+ Assert.assertNotNull(sortRel);
+ Assert.assertTrue(sortRel.isNeedPushToSubCtx());
+ }
+
@Test
public void testCanMatchModelInnerQueryLeft() throws SqlParseException {
// model: TEST_BANK_INCOME inner join TEST_BANK_LOCATION
@@ -127,7 +148,8 @@ public class RealizationChooserTest extends NLocalWithSparkSessionTest {
String sql = "select a.NAME from TEST_BANK_INCOME a left join TEST_BANK_LOCATION b \n"
+ " on a.COUNTRY = b.COUNTRY where " + filter;
OLAPContext olapContext = OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
- Map<String, String> sqlAlias2ModelNameMap = OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
+ Map<String, String> sqlAlias2ModelNameMap = OlapContextTestUtil.matchJoins(dataflow.getModel(),
+ olapContext);
Assert.assertTrue(sqlAlias2ModelNameMap.isEmpty());
}
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
index 094fa8c3c5..e7e2a88b5c 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
@@ -56,6 +56,19 @@ public class OlapContextTestUtil {
return getOlapContexts();
}
+ public static RelNode cutOlapContextsAndReturnRelNode(String project, String sql) throws SqlParseException {
+ QueryExec queryExec = new QueryExec(project, KylinConfig.getInstanceFromEnv());
+ RelNode rel = null;
+ try {
+ rel = queryExec.parseAndOptimize(sql);
+ QueryContextCutter.selectRealization(project, rel, false);
+ } catch (NoRealizationFoundException | NoStreamingRealizationFoundException e) {
+ // When NoRealizationFoundException occurs, do nothing
+ // because we only need to obtain OlapContexts.
+ }
+ return rel;
+ }
+
public static List<OLAPContext> getOlapContexts(String project, String sql, boolean reCutBanned,
Consumer<NoRealizationFoundException> consumer) throws SqlParseException {
QueryExec queryExec = new QueryExec(project, KylinConfig.getInstanceFromEnv());
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
index 2fe32a6fca..9f91b0f355 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapSortRel.java
@@ -18,7 +18,10 @@
package org.apache.kylin.query.relnode;
+import java.util.Collection;
+import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
@@ -26,15 +29,21 @@ import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rex.RexNode;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.query.util.ICutContextStrategy;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import lombok.Getter;
+import lombok.Setter;
public class KapSortRel extends OLAPSortRel implements KapRel {
private Set<OLAPContext> subContexts = Sets.newHashSet();
+ @Setter
+ @Getter
+ protected boolean needPushToSubCtx;
public KapSortRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, RelCollation collation,
RexNode offset, RexNode fetch) {
@@ -101,6 +110,12 @@ public class KapSortRel extends OLAPSortRel implements KapRel {
this.context.addSort(orderCol, order);
this.context.allColumns.addAll(orderCol.getSourceColumns());
}
+ } else if (needPushToSubCtx) {
+ List<Set<TblColRef>> sourceColumns = this.columnRowType.getSourceColumns();
+ if (CollectionUtils.isNotEmpty(sourceColumns)) {
+ ContextUtil.updateSubContexts(
+ sourceColumns.stream().flatMap(Collection::stream).collect(Collectors.toSet()), subContexts);
+ }
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
index 92bfbc4225..bf2c46fc1e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapTableScan.java
@@ -33,16 +33,15 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.query.engine.KECalciteConfig;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.query.schema.OLAPTable;
-import org.apache.kylin.query.engine.KECalciteConfig;
import org.apache.kylin.query.util.ICutContextStrategy;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
/**
*/
public class KapTableScan extends OLAPTableScan implements EnumerableRel, KapRel {
@@ -153,6 +152,9 @@ public class KapTableScan extends OLAPTableScan implements EnumerableRel, KapRel
break;
}
KapRel parent = (KapRel) tempParent;
+ if (parent instanceof KapSortRel) {
+ ((KapSortRel) parent).setNeedPushToSubCtx(true);
+ }
if (topProjParent == null && parent instanceof OLAPProjectRel
&& !((OLAPProjectRel) parent).isMerelyPermutation()) {
(kylin) 02/35: KYLIN-5793 Fix Chinese gibberish when Spring ${} attribute injection
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 07354967d424f3ba76d87ea562948f8b5f69a3a5
Author: Liang.Hua <36...@users.noreply.github.com>
AuthorDate: Mon Aug 28 14:56:11 2023 +0800
KYLIN-5793 Fix Chinese gibberish when Spring ${} attribute injection
---
.../org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 3f30693497..1f715fb8bb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -62,6 +62,7 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
}
InputStream is = IOUtils.toInputStream(propString, Charset.defaultCharset());
resources[0] = new InputStreamResource(is);
+ this.setFileEncoding(Charset.defaultCharset().toString());
this.setLocations(resources);
}
(kylin) 17/35: KYLIN-5811 Enhancement of SCD2
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 5092e7a239146305197ad6fb5fda28fdc090b376
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Tue Jun 6 11:29:51 2023 +0800
KYLIN-5811 Enhancement of SCD2
1. scd2 support join pairs of (>=, <=) and (>, <=)
2. scd2 support recommendation
3. correct toString and compare of NonEquiJoinCondition
4. flat-table sql fix & add a setting to control suggest scd2 model
Support semantically equivalent scd2 join conditions.
For example: "a >= b and c > a" is equivalent to "a >= b and a < c".
---
.../org/apache/kylin/common/KylinConfigBase.java | 8 +
.../apache/kylin/common/debug/BackdoorToggles.java | 5 -
.../kylin/common/exception/QueryErrorCode.java | 6 -
.../common/exception/code/ErrorCodeServer.java | 9 +
.../org/apache/kylin/common/util/ThreadUtil.java | 3 +
.../resources/kylin_error_msg_conf_cn.properties | 9 +
.../resources/kylin_error_msg_conf_en.properties | 9 +
.../main/resources/kylin_errorcode_conf.properties | 9 +
.../kylin/metadata/cube/cuboid/IndexMatcher.java | 7 +-
.../org/apache/kylin/metadata/model/JoinDesc.java | 30 +-
.../apache/kylin/metadata/model/NDataModel.java | 4 +-
.../kylin/metadata/model/NonEquiJoinCondition.java | 301 ++++++++++++--
.../metadata/model/NonEquiJoinConditionType.java | 24 --
.../apache/kylin/metadata/model/TableExtDesc.java | 5 +
.../metadata/model/{tool => }/TruthTable.java | 184 ++++-----
.../model/tool/JoinDescNonEquiCompBean.java | 60 ---
.../model/tool/NonEquiJoinConditionComparator.java | 166 --------
.../model/tool/NonEquiJoinConditionVisitor.java | 59 ---
.../metadata/model/tool/TypedLiteralConverter.java | 76 ----
.../kylin/metadata/model/util/JoinDescUtil.java | 7 +-
.../metadata/model/util/scd2/SCD2CondChecker.java | 88 ++--
.../metadata/model/util/scd2/SCD2Exception.java | 25 --
.../util/scd2/SCD2NonEquiCondSimplification.java | 201 ---------
.../metadata/model/util/scd2/SCD2SqlConverter.java | 4 +-
.../metadata/model/util/scd2/Scd2Simplifier.java | 174 ++++++++
.../model/util/scd2/SimplifiedJoinDesc.java | 23 +-
.../kylin/metadata/project/NProjectManager.java | 10 +-
.../kylin/query/util/QueryInterruptChecker.java | 23 +-
.../kylin/metadata/model/ModelNonEquiCondMock.java | 8 +-
.../metadata/model/NeqEquiJoinConditionTest.java | 281 +++++++++++++
.../tool/NonEquiJoinConditionComparatorTest.java | 234 -----------
.../apache/kylin/rest/service/TableService.java | 13 +-
.../org/apache/kylin/util/OlapContextTestUtil.java | 6 +-
.../controller/NUserGroupControllerV2Test.java | 2 +-
.../kylin/rest/service/ModelSemanticHelper.java | 58 +--
.../rest/util/SCD2SimplificationConvertUtil.java | 30 +-
.../kylin/rest/request/ModelRequestTest.java | 6 +-
.../apache/kylin/rest/service/BaseIndexTest.java | 7 +-
.../rest/service/ModelServiceBrokenRepairTest.java | 1 +
.../service/ModelServiceSemanticUpdateTest.java | 456 ++++++++++-----------
.../kylin/rest/service/ModelServiceTest.java | 31 +-
.../service/ModelTdsServiceColumnNameTest.java | 2 +-
.../kylin/rest/service/ModelTdsServiceTest.java | 13 +-
.../kylin/rest/service/TableServiceTest.java | 2 +-
.../kylin/query/relnode/KapNonEquiJoinRel.java | 29 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 32 +-
.../kyligence/kap/query/optrule/KapJoinRule.java | 242 ++++++++---
.../apache/kylin/query/engine/PlannerFactory.java | 12 +-
.../org/apache/kylin/query/engine/QueryExec.java | 6 +-
.../engine/exec/sparder/SparderQueryPlanExec.java | 13 +-
.../kylin/query/util/QueryContextCutter.java | 22 +-
.../kylin/query/engine/QueryRoutingEngineTest.java | 3 +-
.../kylin/query/engine/SelectRealizationTest.java | 6 +-
.../kylin/query/pushdown/SparkSqlClient.scala | 7 +-
.../kylin/query/runtime/plan/TableScanPlan.scala | 6 +-
.../kylin/query/util/SparderDerivedUtil.scala | 12 +-
.../org/apache/kylin/tool/MaintainModeTool.java | 4 +-
.../kylin/tool/routine/MaintainModeToolTest.java | 40 +-
58 files changed, 1587 insertions(+), 1526 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 954d13a685..e991f4d990 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2075,6 +2075,10 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.non-equi-join-model-enabled", FALSE));
}
+ public boolean isNonEquiJoinRecommendationEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.model.non-equi-join-recommendation-enabled", FALSE));
+ }
+
public String getQueryAccessController() {
return getOptional("kylin.query.access-controller", null);
}
@@ -4128,4 +4132,8 @@ public abstract class KylinConfigBase implements Serializable {
public String getFileSegmentSuccessFile() {
return getOptional("kylin.fileseg.success-file", null);
}
+
+ public boolean isPrintQueryPlanEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.print-logical-plan", FALSE));
+ }
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 2944265128..57c3513562 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -22,7 +22,6 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.Pair;
-
import org.apache.kylin.guava30.shaded.common.collect.Maps;
/**
@@ -77,10 +76,6 @@ public class BackdoorToggles {
return getBoolean(QUERY_FROM_AUTO_MODELING);
}
- public static boolean getIsQueryNonEquiJoinModelEnabled() {
- return getBoolean(QUERY_NON_EQUI_JOIN_MODEL_ENABLED);
- }
-
public static String getPartitionDumpDir() {
return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java
index 96e305316c..a89ee7945d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java
@@ -21,12 +21,6 @@ package org.apache.kylin.common.exception;
public enum QueryErrorCode implements ErrorCodeSupplier {
// 20002XXX model
- SCD2_DUPLICATE_JOIN_COL("KE-020002001"), //
- SCD2_DUPLICATE_FK_PK_PAIR("KE-020002002"), //
- SCD2_EMPTY_EQUI_JOIN("KE-020002003"), //
- SCD2_DUPLICATE_CONDITION("KE-020002004"), //
- SCD2_COMMON_ERROR("KE-020002005"), //
- SCD2_SAVE_MODEL_WHEN_DISABLED("KE-020002006"), //
CC_EXPRESSION_ILLEGAL("KE-020002007"),
// 20003XXX user
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index f2562e3a9f..16f290a6f1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -22,6 +22,7 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
// 100012XX project
PROJECT_NOT_EXIST("KE-010001201"),
PROJECT_MULTI_PARTITION_DISABLE("KE-010001208"),
+ PROJECT_SCD2_IS_NOT_ALLOWED("KE-010001209"),
// 100022XX model
MODEL_NOT_EXIST("KE-010002201"),
@@ -39,6 +40,14 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_NAME_TOO_LONG("KE-010002305"),
MODEL_SECOND_STORAGE_PARTITION_INVALID("KE-010002306"),
PARTITION_SECOND_STORAGE_PARTITION_INVALID("KE-010002307"),
+ SCD2_MODEL_CAN_ONLY_CONNECT_BY_AND("KE-010002401"),
+ SCD2_MODEL_CONTAINS_ILLEGAL_EXPRESSIONS("KE-010002402"),
+ SCD2_CONDITION_MUST_APPEAR_IN_PAIRS("KE-010002403"),
+ SCD2_MODEL_REQUIRES_AT_LEAST_ONE_EQUAL_CONDITION("KE-010002404"),
+ SCD2_MODEL_REQUIRES_AT_LEAST_ONE_NON_EQUAL_CONDITION("KE-010002405"),
+ SCD2_MODEL_PK_FK_UNIQUE_CHECK_FAILED("KE-010002406"),
+ DUPLICATE_MODEL_JOIN_CONDITIONS("KE-010002407"),
+ SCD2_MODEL_UNKNOWN_EXCEPTION("KE-010002408"),
// 100252XX Cube
CUBE_NOT_EXIST("KE-010025201"),
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
index 24b3d67a38..fc4c666333 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
@@ -40,4 +40,7 @@ public class ThreadUtil {
.collect(Collectors.toList()), "\n");
}
+ public static void warnKylinStackTrace(String simpleErrorMsg) {
+ log.warn(simpleErrorMsg + getKylinStackTrace());
+ }
}
\ No newline at end of file
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 408746be99..d76a53999c 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -22,6 +22,7 @@ KE-030001201=无法构建，索引“%s” 的预期行数与实际构建行数
## 100012XX 项目
KE-010001201=找不到项目 “%s”。请检查后重试。
KE-010001208=无法使用，项目 “%s” 未开启多级分区。请开启后再试。
+KE-010001209=请在高级设置中开启拉链表功能。
## 100022XX 模型
KE-010002201=无法找到模型，请检查后再试。
@@ -39,6 +40,14 @@ KE-010002304=SUM_LC度量的时间类型 '%s' 不合法。
KE-010002305=模型名称最长 127 字符，请修改后重试。
KE-010002306=无法保存模型。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
KE-010002307=无法保存分区设置。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
+KE-010002401=拉链表需要两个非等值的关联，并且必须用AND操作符相连。
+KE-010002402=在非等值关联中有非法表达式。
+KE-010002403=非等值关联必须成对出现。
+KE-010002404=拉链表至少需要一个“=”连接的等值关联。
+KE-010002405=拉链表至少需要一对非等值的关联。
+KE-010002406=主键/外键唯一性检查失败。
+KE-010002407=存在重复的关联关系。
+KE-010002408=拉链表未知错误：%s
## 100252XX Cube
KE-010025201=无法找到相关 Cube。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 4bdfe174ba..59772faaad 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -22,6 +22,7 @@ KE-030001201=Can't build, the expected number of rows for index "%s" does not ma
## 100012XX project
KE-010001201=Can't find project "%s". Please check and try again.
KE-010001208=Can't use, the multilevel partitioning is not enabled for project “%s”. Please turn it on and try again.
+KE-010001209=Please first turn on history table setting in Advanced Settings.
## 100022XX model
KE-010002201=Can't find model. Please check and try again.
@@ -39,6 +40,14 @@ KE-010002304=SUM_LC Measure's time column type '%s' is illegal.
KE-010002305=The maximum length of the model name is 127 characters, please modify and try again.
KE-010002306=Can't save the model. When the model uses incremental load method and the tiered storage is ON, the time partition column must be added as a dimension.
KE-010002307=Can't save the model partition. When the model uses incremental load method and the tiered storage is ON, the time partition column must be added as a dimension.
+KE-010002401=History zipper table must have two non-equal join relationships linked by AND operator.
+KE-010002402=There are illegal expressions in non-equal join relationships.
+KE-010002403=Non-equal join relationships must be used in pairs.
+KE-010002404=Join relationships in history zipper table should have at least one `=` operator.
+KE-010002405=History zipper table should have at least two non-equal join conditions in pairs.
+KE-010002406=The unique check of primary key/foreign key failed.
+KE-010002407=Duplicate join condition.
+KE-010002408==History zipper table unknown exception: %s
## 100252XX Cube
KE-010025201=Can't find the cube.
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index f0162f4e95..2f28e7bfa6 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -23,6 +23,7 @@ KE-030001201
# Server
## 100012XX project
KE-010001208
+KE-010001209
## 100022XX model
KE-010002201
@@ -40,6 +41,14 @@ KE-010002304
KE-010002305
KE-010002306
KE-010002307
+KE-010002401
+KE-010002402
+KE-010002403
+KE-010002404
+KE-010002405
+KE-010002406
+KE-010002407
+KE-010002408
## 100252XX Cube
KE-010025201
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java
index eea05e1a8e..5db2a673ec 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/IndexMatcher.java
@@ -43,7 +43,7 @@ import org.apache.kylin.metadata.model.DeriveInfo;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.util.scd2.SCD2NonEquiCondSimplification;
+import org.apache.kylin.metadata.model.util.scd2.Scd2Simplifier;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -227,9 +227,8 @@ public abstract class IndexMatcher {
}
private boolean matchNonEquiJoinFks(final IndexEntity indexEntity, final JoinDesc joinDesc) {
- return joinDesc.isNonEquiJoin() && indexEntity.dimensionsDerive(
- Stream.of(SCD2NonEquiCondSimplification.INSTANCE.extractFksFromNonEquiJoinDesc(joinDesc))
- .map(tblColMap::get).collect(Collectors.toList()));
+ return joinDesc.isNonEquiJoin() && indexEntity.dimensionsDerive(Stream
+ .of(Scd2Simplifier.INSTANCE.extractNeqFks(joinDesc)).map(tblColMap::get).collect(Collectors.toList()));
}
@Getter
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index fcda2f0e01..be8e2e183b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -26,11 +26,12 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
@@ -310,4 +311,29 @@ public class JoinDesc implements Serializable {
}
return joinKey;
}
+
+ public static class NonEquivJoinDesc extends JoinDesc {
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() * 31 + Objects.hashCode(getNonEquiJoinCondition());
+ }
+
+ @Override
+ public boolean equals(Object o2) {
+ if (Objects.isNull(o2) || !(o2 instanceof NonEquivJoinDesc)) {
+ return false;
+ }
+
+ JoinDesc nonEquivJoin = ((NonEquivJoinDesc) o2);
+
+ // check equiv-join part
+ if (!super.equals(nonEquivJoin)) {
+ return false;
+ }
+
+ //check non-equiv-join part
+ return Objects.equals(this.getNonEquiJoinCondition(), nonEquivJoin.getNonEquiJoinCondition());
+ }
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index ec47608b85..183b5ebc59 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -66,8 +66,8 @@ import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.model.PartitionDesc.PartitionType;
+import org.apache.kylin.metadata.model.graph.JoinsGraph;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -923,7 +923,7 @@ public class NDataModel extends RootPersistentEntity {
return;
}
- if (cond.getType() == NonEquiJoinConditionType.COLUMN) {
+ if (cond.getType() == NonEquiJoinCondition.Type.COLUMN) {
cond.setColRef(findColumn(cond.getValue()));
}
if (cond.getOperands().length > 0) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinCondition.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinCondition.java
index a8b96288d0..a40a406e3f 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinCondition.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinCondition.java
@@ -19,37 +19,60 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Objects;
import java.util.Set;
-import java.util.stream.Collectors;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
-import org.apache.kylin.metadata.model.tool.NonEquiJoinConditionComparator;
-import org.apache.kylin.metadata.model.tool.TypedLiteralConverter;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.TimeString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
@SuppressWarnings("serial")
+@Slf4j
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class NonEquiJoinCondition implements Serializable {
+ private static final Map<SqlKind, TruthTable.Operator> OP_TRUTH_MAPPING = ImmutableMap.<SqlKind, TruthTable.Operator> builder()
+ .put(SqlKind.AND, TruthTable.Operator.AND) //
+ .put(SqlKind.OR, TruthTable.Operator.OR) //
+ .put(SqlKind.NOT, TruthTable.Operator.NOT) //
+ .build();
+
+ private static final Map<SqlKind, SqlKind> OP_INVERSE_MAPPING = ImmutableMap.<SqlKind, SqlKind> builder()
+ .put(SqlKind.NOT_EQUALS, SqlKind.EQUALS) //
+ .put(SqlKind.NOT_IN, SqlKind.IN) //
+ .put(SqlKind.IS_NOT_NULL, SqlKind.IS_NULL) //
+ .put(SqlKind.IS_NOT_FALSE, SqlKind.IS_FALSE) //
+ .put(SqlKind.IS_NOT_TRUE, SqlKind.IS_TRUE) //
+ .put(SqlKind.IS_NOT_DISTINCT_FROM, SqlKind.IS_DISTINCT_FROM) //
+ .put(SqlKind.LESS_THAN, SqlKind.GREATER_THAN_OR_EQUAL) //
+ .put(SqlKind.LESS_THAN_OR_EQUAL, SqlKind.GREATER_THAN) //
+ .build();
+
@Getter
@Setter
@JsonProperty("type")
- private NonEquiJoinConditionType type;
+ private Type type;
@Getter
@Setter
@@ -88,6 +111,9 @@ public class NonEquiJoinCondition implements Serializable {
public NonEquiJoinCondition() {
}
+ static final Set<SqlKind> EXCHANGEABLE_OPERATOR = Sets.newHashSet(SqlKind.AND, SqlKind.OR, SqlKind.EQUALS,
+ SqlKind.NOT_EQUALS, SqlKind.IN, SqlKind.NOT_IN);
+
public NonEquiJoinCondition(SqlOperator op, NonEquiJoinCondition[] operands, RelDataType dataType) {
this(op.getName(), op.getKind(), operands, new DataType(dataType));
}
@@ -104,20 +130,20 @@ public class NonEquiJoinCondition implements Serializable {
this.opName = opName;
this.op = op;
this.operands = operands;
- this.type = NonEquiJoinConditionType.EXPRESSION;
+ this.type = Type.EXPRESSION;
this.dataType = dataType;
}
public NonEquiJoinCondition(String value, DataType dataType) {
this.op = SqlKind.LITERAL;
- this.type = NonEquiJoinConditionType.LITERAL;
+ this.type = Type.LITERAL;
this.value = value;
this.dataType = dataType;
}
public NonEquiJoinCondition(TblColRef tblColRef, DataType dataType) {
this.op = SqlKind.INPUT_REF;
- this.type = NonEquiJoinConditionType.COLUMN;
+ this.type = Type.COLUMN;
this.value = tblColRef.getIdentity();
this.colRef = tblColRef;
this.dataType = dataType;
@@ -140,18 +166,89 @@ public class NonEquiJoinCondition implements Serializable {
return condCopy;
}
+ public NonEquiJoinCondition[] getSortedOperands() {
+ if (EXCHANGEABLE_OPERATOR.contains(op)) {
+ return Arrays.stream(operands).sorted(Comparator.comparing(NonEquiJoinCondition::getOp))
+ .toArray(NonEquiJoinCondition[]::new);
+ } else {
+ return Arrays.copyOf(operands, operands.length);
+ }
+ }
+
+ private TruthTable<NonEquiJoinCondition> createTruthTable() {
+ TruthTable.ExprBuilder<NonEquiJoinCondition> builder = new TruthTable.ExprBuilder<>(
+ new NeqCondOperandComparator());
+ this.accept(builder);
+ TruthTable.Expr<NonEquiJoinCondition> builtExpr = builder.build();
+ return new TruthTable<>(new ArrayList<>(builder.allOperandSet), builtExpr);
+ }
+
+ private void accept(TruthTable.ExprBuilder<NonEquiJoinCondition> builder) {
+ switch (op) {
+ case AND:
+ case NOT:
+ case OR:
+ builder.compositeStart(OP_TRUTH_MAPPING.get(op));
+ for (NonEquiJoinCondition operand : operands) {
+ operand.accept(builder);
+ }
+ builder.compositeEnd();
+ break;
+ default:
+ this.addOperand(builder);
+ break;
+ }
+ }
+
+ private void addOperand(TruthTable.ExprBuilder<NonEquiJoinCondition> builder) {
+ NonEquiJoinCondition copied = this.copy();
+ if (inverseCondOperator(copied)) {
+ builder.compositeStart(TruthTable.Operator.NOT);
+ normalizedCondOperandOrderings(copied);
+ builder.addExpr(copied);
+ builder.compositeEnd();
+ } else {
+ normalizedCondOperandOrderings(copied);
+ builder.addExpr(copied);
+ }
+ }
+
+ private boolean inverseCondOperator(NonEquiJoinCondition neqJoinCond) {
+ if (OP_INVERSE_MAPPING.containsKey(neqJoinCond.getOp())) {
+ neqJoinCond.setOp(OP_INVERSE_MAPPING.get(neqJoinCond.getOp()));
+ neqJoinCond.setOpName(neqJoinCond.getOp().sql);
+ return true;
+ }
+ return false;
+ }
+
+ private void normalizedCondOperandOrderings(NonEquiJoinCondition neqJoinCond) {
+ // if operands ordering does not matter, sort by operands' digest
+ // case =, <>
+ if (neqJoinCond.getOp() == SqlKind.EQUALS) {
+ Arrays.sort(neqJoinCond.getOperands(), Comparator.comparing(NonEquiJoinCondition::toString));
+ }
+
+ // sort IN args
+ if (neqJoinCond.getOp() == SqlKind.IN) {
+ Arrays.sort(neqJoinCond.getOperands(), 1, neqJoinCond.getOperands().length,
+ Comparator.comparing(NonEquiJoinCondition::toString));
+ }
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(op);
sb.append("(");
- for (NonEquiJoinCondition input : operands) {
+ NonEquiJoinCondition[] sorted = getSortedOperands();
+ for (NonEquiJoinCondition input : sorted) {
sb.append(input.toString());
sb.append(", ");
}
- if (type == NonEquiJoinConditionType.LITERAL) {
+ if (type == Type.LITERAL) {
sb.append(value);
- } else if (type == NonEquiJoinConditionType.COLUMN) {
+ } else if (type == Type.COLUMN) {
if (colRef != null) {
sb.append(colRef.getColumnWithTableAndSchema());
} else {
@@ -164,8 +261,18 @@ public class NonEquiJoinCondition implements Serializable {
@Override
public boolean equals(Object obj) {
- return obj != null && obj instanceof NonEquiJoinCondition
- && NonEquiJoinConditionComparator.equals(this, (NonEquiJoinCondition) obj);
+ if (!(obj instanceof NonEquiJoinCondition)) {
+ return false;
+ }
+
+ NonEquiJoinCondition neqJoinCond = (NonEquiJoinCondition) obj;
+ try {
+ return Objects.equals(this.createTruthTable(), neqJoinCond.createTruthTable());
+ } catch (Exception e) {
+ log.error("Error on comparing condition({}), condition({}). Error Msg: {}", this, neqJoinCond,
+ e.getMessage());
+ return false;
+ }
}
@Override
@@ -183,9 +290,9 @@ public class NonEquiJoinCondition implements Serializable {
result = prime * result + operand.hashCode();
}
- if (type == NonEquiJoinConditionType.LITERAL) {
+ if (type == Type.LITERAL) {
result = prime * result + value.hashCode();
- } else if (type == NonEquiJoinConditionType.COLUMN) {
+ } else if (type == Type.COLUMN) {
if (colRef != null) {
result = prime * result + colRef.hashCode();
} else {
@@ -195,25 +302,10 @@ public class NonEquiJoinCondition implements Serializable {
return result;
}
- public Set<String> getAllReferencingColumns() {
- return doGetAllReferencingColumns(this);
- }
-
- private Set<String> doGetAllReferencingColumns(NonEquiJoinCondition cond) {
- if (cond.type == NonEquiJoinConditionType.COLUMN) {
- return Sets.newHashSet(cond.colRef.getIdentity());
- } else if (cond.type == NonEquiJoinConditionType.EXPRESSION) {
- return Arrays.stream(cond.operands).map(this::doGetAllReferencingColumns).flatMap(Collection::stream)
- .collect(Collectors.toSet());
- } else {
- return new HashSet<>();
- }
- }
-
@Setter
@Getter
@NoArgsConstructor
- public static final class SimplifiedNonEquiJoinCondition implements Serializable {
+ public static final class SimplifiedJoinCondition implements Serializable {
private static final long serialVersionUID = -1577556052145832500L;
@@ -226,14 +318,13 @@ public class NonEquiJoinCondition implements Serializable {
@JsonProperty("op")
private SqlKind op;
- public SimplifiedNonEquiJoinCondition(String foreignKey, String primaryKey, SqlKind op) {
+ public SimplifiedJoinCondition(String foreignKey, String primaryKey, SqlKind op) {
this.foreignKey = foreignKey;
this.primaryKey = primaryKey;
this.op = op;
}
- public SimplifiedNonEquiJoinCondition(String foreignKey, TblColRef fk, String primaryKey, TblColRef pk,
- SqlKind op) {
+ public SimplifiedJoinCondition(String foreignKey, TblColRef fk, String primaryKey, TblColRef pk, SqlKind op) {
this.foreignKey = foreignKey;
this.primaryKey = primaryKey;
this.op = op;
@@ -246,5 +337,143 @@ public class NonEquiJoinCondition implements Serializable {
@JsonIgnore
private TblColRef pk;
+
+ public String displaySql() {
+ return fk.getDoubleQuoteExp() + " " + op.sql + " " + pk.getDoubleQuoteExp();
+ }
+ }
+
+ static class NeqCondOperandComparator implements Comparator<NonEquiJoinCondition> {
+ @Override
+ public int compare(NonEquiJoinCondition cond1, NonEquiJoinCondition cond2) {
+ if (!(Objects.equals(cond1.getOp(), cond2.getOp())
+ && cond1.getOperands().length == cond2.getOperands().length
+ && Objects.equals(cond1.getType(), cond2.getType())
+ && Objects.equals(cond1.getDataType(), cond2.getDataType()))) {
+ return 1;
+ }
+
+ // compare opName on for SqlKind OTHER
+ if (cond1.getOp() == SqlKind.OTHER || cond1.getOp() == SqlKind.OTHER_FUNCTION
+ && (!Objects.equals(cond1.getOpName(), cond2.getOpName()))) {
+ return 1;
+ }
+
+ if (cond1.getType() == Type.LITERAL) {
+ return Objects.equals(cond1.getValue(), cond2.getValue()) ? 0 : 1;
+ } else if (cond1.getType() == Type.COLUMN) {
+ return Objects.equals(cond1.getColRef().getColumnDesc(), cond2.getColRef().getColumnDesc()) ? 0 : 1;
+ }
+ NonEquiJoinCondition[] sorted1 = cond1.getSortedOperands();
+ NonEquiJoinCondition[] sorted2 = cond2.getSortedOperands();
+ for (int i = 0; i < sorted1.length; i++) {
+ if (compare(sorted1[i], sorted2[i]) != 0) {
+ return 1;
+ }
+ }
+ return 0;
+ }
+ }
+
+ public interface NeqConditionVisitor {
+
+ default NonEquiJoinCondition visit(NonEquiJoinCondition neqJoinCond) {
+ if (neqJoinCond == null) {
+ return null;
+ }
+
+ if (neqJoinCond.getType() == Type.LITERAL) {
+ return visitLiteral(neqJoinCond);
+ } else if (neqJoinCond.getType() == Type.COLUMN) {
+ return visitColumn(neqJoinCond);
+ } else if (neqJoinCond.getType() == Type.EXPRESSION) {
+ return visitExpression(neqJoinCond);
+ } else {
+ return visit(neqJoinCond);
+ }
+ }
+
+ default NonEquiJoinCondition visitExpression(NonEquiJoinCondition neqJoinCond) {
+ NonEquiJoinCondition[] ops = new NonEquiJoinCondition[neqJoinCond.getOperands().length];
+ for (int i = 0; i < neqJoinCond.getOperands().length; i++) {
+ ops[i] = visit(neqJoinCond.getOperands()[i]);
+ }
+ neqJoinCond.setOperands(ops);
+ return neqJoinCond;
+ }
+
+ default NonEquiJoinCondition visitColumn(NonEquiJoinCondition neqJoinCond) {
+ return neqJoinCond;
+ }
+
+ default NonEquiJoinCondition visitLiteral(NonEquiJoinCondition neqJoinCond) {
+ return neqJoinCond;
+ }
+ }
+
+ public static class TypedLiteralConverter {
+ private TypedLiteralConverter() {
+ }
+
+ static final String NULL = "KY_LITERAL_NULL";
+
+ public static Object stringValueToTypedValue(String value, DataType dataType) {
+ if (value.equals(NULL)) {
+ return null;
+ }
+ switch (dataType.getTypeName()) {
+ case DECIMAL:
+ return new BigDecimal(value);
+ case BIGINT:
+ return Long.parseLong(value);
+ case SMALLINT:
+ return Short.parseShort(value);
+ case TINYINT:
+ return Byte.parseByte(value);
+ case INTEGER:
+ return Integer.parseInt(value);
+ case DOUBLE:
+ return Double.parseDouble(value);
+ case FLOAT:
+ case REAL:
+ return Float.parseFloat(value);
+ case DATE:
+ return DateString.fromDaysSinceEpoch(Integer.parseInt(value));
+ case TIME:
+ case TIME_WITH_LOCAL_TIME_ZONE:
+ return TimeString.fromMillisOfDay(Integer.parseInt(value));
+ case TIMESTAMP:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return TimestampString.fromMillisSinceEpoch(Long.parseLong(value));
+ case BOOLEAN:
+ return Boolean.parseBoolean(value);
+ case CHAR:
+ case VARCHAR:
+ default:
+ return value;
+ }
+ }
+
+ public static String typedLiteralToString(RexLiteral literal) {
+ if (literal.getValue3() == null) {
+ return NULL;
+ }
+ return String.valueOf(literal.getValue3());
+ }
+ }
+
+ public enum Type {
+ /**
+ * A sub NonEquiJoinCondition expression.
+ */
+ EXPRESSION,
+ /**
+ * A column.
+ */
+ COLUMN,
+ /**
+ * A literal, unlikely type for scd2 join condition.
+ */
+ LITERAL
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java
deleted file mode 100644
index 6bc8d5fd4f..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NonEquiJoinConditionType.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model;
-
-public enum NonEquiJoinConditionType {
- EXPRESSION, // expression with other NonEquiJoinCondition as operands
- COLUMN, LITERAL
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index aed99e2747..f30c52a4b2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -157,6 +157,11 @@ public class TableExtDesc extends RootPersistentEntity implements Serializable {
+ MetadataConstants.FILE_SURFIX;
}
+ @Override
+ public List<String> getLockPaths(String ignored) {
+ return getTableDesc().getLockPaths();
+ }
+
public void addDataSourceProp(String key, String value) {
this.dataSourceProps.put(key, value);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TruthTable.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TruthTable.java
similarity index 54%
rename from src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TruthTable.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TruthTable.java
index 6dcd3d3beb..b16c0adec2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TruthTable.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TruthTable.java
@@ -16,11 +16,10 @@
* limitations under the License.
*/
-package org.apache.kylin.metadata.model.tool;
+package org.apache.kylin.metadata.model;
import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashMap;
@@ -33,106 +32,114 @@ import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-public class TruthTable {
+import lombok.extern.slf4j.Slf4j;
- private static Logger logger = LoggerFactory.getLogger(TruthTable.class);
+@Slf4j
+public class TruthTable<E> {
- private List<Expr> allOperands;
- private Expr expr;
+ private final List<Expr<E>> allOperands;
+ private final Expr<E> expr;
- private TruthTable(List<Expr> allOperands, Expr expr) {
+ TruthTable(List<Expr<E>> allOperands, Expr<E> expr) {
this.allOperands = allOperands;
this.expr = expr;
}
- public static boolean equals(TruthTable tbl1, TruthTable tbl2) {
- logger.debug("Comparing table1: {} with table2 {}", tbl1, tbl2);
- if (tbl1.allOperands.size() != tbl2.allOperands.size()) {
+ @Override
+ public String toString() {
+ return "TruthTable{" + "allOperands=" + allOperands + ", expr=" + expr + '}';
+ }
+
+ @Override
+ public int hashCode() {
+ return allOperands.hashCode() * 31 + expr.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof TruthTable)) {
return false;
}
- for (Expr operand : tbl1.allOperands) {
+ TruthTable<E> tbl2 = (TruthTable<E>) obj;
+ log.debug("Comparing table1: {} with table2 {}", this, tbl2);
+ if (this.allOperands.size() != tbl2.allOperands.size()) {
+ return false;
+ }
+ for (Expr<E> operand : this.allOperands) {
if (!tbl2.allOperands.contains(operand)) {
return false;
}
}
- InputGenerator tbl1InputGenerator = tbl1.createInputGenerator();
+ InputGenerator<E> tbl1InputGenerator = new InputGenerator<>();
while (tbl1InputGenerator.hasNext()) {
- Input input = tbl1InputGenerator.next();
- if (tbl1.eval(input) != tbl2.eval(input.mapOperands(tbl2.allOperands))) {
+ Input<E> input = tbl1InputGenerator.next();
+ if (this.eval(input) != tbl2.eval(input.mapOperands(tbl2.allOperands))) {
return false;
}
}
return true;
}
- public boolean eval(Input input) {
+ public boolean eval(Input<E> input) {
return doEval(expr, input);
}
- private boolean doEval(Expr expr, Input input) {
- switch (expr.operator) {
+ private boolean doEval(Expr<E> expr, Input<E> input) {
+ switch (expr.op) {
case IDENTITY:
return input.getValue(expr);
case NOT:
- return !doEval(expr.exprs[0], input);
+ return !doEval(expr.operands.get(0), input);
case AND:
- for (Expr innerExpr : expr.exprs) {
+ for (Expr<E> innerExpr : expr.operands) {
if (!doEval(innerExpr, input)) {
return false;
}
}
return true;
case OR:
- for (Expr innerExpr : expr.exprs) {
+ for (Expr<E> innerExpr : expr.operands) {
if (doEval(innerExpr, input)) {
return true;
}
}
return false;
default:
- throw new IllegalStateException("Invalid Operator" + expr.operator);
+ throw new IllegalStateException("Invalid Operator" + expr.op);
}
}
- private static class Input {
- private Map<Expr, Boolean> inputValues;
+ private static class Input<T> {
+ private final Map<Expr<T>, Boolean> inputValues;
- Input(Map<Expr, Boolean> inputValues) {
+ Input(Map<Expr<T>, Boolean> inputValues) {
this.inputValues = inputValues;
}
- boolean getValue(Expr expr) {
+ boolean getValue(Expr<T> expr) {
return inputValues.get(expr);
}
/**
* map input with operands in different orderings
- * @param operands
- * @return
*/
- Input mapOperands(List<Expr> operands) {
+ Input<T> mapOperands(List<Expr<T>> operands) {
Preconditions.checkArgument(operands.size() == inputValues.size());
- Map<Expr, Boolean> mappedInput = new HashMap<>();
- for (Expr expr : operands) {
+ Map<Expr<T>, Boolean> mappedInput = new HashMap<>();
+ for (Expr<T> expr : operands) {
Preconditions.checkArgument(inputValues.get(expr) != null, "Invalid table expr for operands mapping");
mappedInput.put(expr, inputValues.get(expr));
}
Preconditions.checkArgument(mappedInput.size() == inputValues.size());
- return new Input(mappedInput);
+ return new Input<>(mappedInput);
}
}
- public InputGenerator createInputGenerator() {
- return new InputGenerator();
- }
-
- public class InputGenerator implements Iterator<Input> {
+ private class InputGenerator<T> implements Iterator<Input<T>> {
int inputSize = allOperands.size();
int inputMax = (int) Math.pow(2, inputSize);
@@ -144,59 +151,60 @@ public class TruthTable {
}
@Override
- public Input next() {
+ public Input<T> next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
- Map<Expr, Boolean> input = new HashMap<>();
- if (logger.isTraceEnabled()) {
- logger.trace("Generating next input {}, max inputValues bits {}", Integer.toBinaryString(inputBits),
+ Map<Expr<T>, Boolean> input = new HashMap<>();
+ if (log.isTraceEnabled()) {
+ log.trace("Generating next input {}, max inputValues bits {}", Integer.toBinaryString(inputBits),
Integer.toBinaryString(inputMax - 1));
}
for (int i = 0; i < inputSize; i++) {
boolean value = (inputBits & (1 << i)) != 0;
- input.put(allOperands.get(i), value);
+ input.put((Expr<T>) allOperands.get(i), value);
}
inputBits++;
- return new Input(input);
+ return new Input<>(input);
}
}
- enum Operator {
+ public enum Operator {
AND, OR, NOT, IDENTITY
}
- private static class Expr<T> {
- Operator operator;
- Expr[] exprs = new Expr[0];
- T operandRef;
+ static class Expr<T> {
+ Operator op;
+ List<Expr<T>> operands;
+ T operand;
Comparator<T> operandComparator;
- Expr(Operator operator, Expr[] exprs, Comparator<T> operandComparator) {
- this.operator = operator;
- this.exprs = exprs;
+ Expr(Operator op, Comparator<T> operandComparator, List<Expr<T>> operands) {
+ this.op = op;
+ this.operands = operands;
this.operandComparator = operandComparator;
}
- Expr(T operandRef, Comparator<T> operandComparator) {
- this.operator = Operator.IDENTITY;
- this.operandRef = operandRef;
+ Expr(T operand, Comparator<T> operandComparator) {
+ this.op = Operator.IDENTITY;
+ this.operand = operand;
this.operandComparator = operandComparator;
+ this.operands = Collections.emptyList();
}
@Override
public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof Expr)) {
+ if (!(obj instanceof Expr)) {
return false;
}
- Expr that = (Expr) obj;
- if (!(operator == that.operator && exprs.length == that.exprs.length
- && operandComparator.compare(operandRef, (T) that.operandRef) == 0)) {
+ Expr<T> that = (Expr<T>) obj;
+ if (!(op == that.op && operands.size() == that.operands.size()
+ && operandComparator.compare(operand, that.operand) == 0)) {
return false;
}
- for (int i = 0; i < exprs.length; i++) {
- if (!Objects.equals(exprs[i], that.exprs[i])) {
+ for (int i = 0; i < operands.size(); i++) {
+ if (!Objects.equals(operands.get(i), that.operands.get(i))) {
return false;
}
}
@@ -207,9 +215,9 @@ public class TruthTable {
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + operator.hashCode();
- result = prime * result + operandRef.hashCode();
- for (Expr expr : exprs) {
+ result = prime * result + op.hashCode();
+ result = prime * result + operand.hashCode();
+ for (Expr<T> expr : operands) {
result = prime * result + expr.hashCode();
}
return result;
@@ -217,32 +225,15 @@ public class TruthTable {
@Override
public String toString() {
- return "Expr{" + "operator=" + operator + ", exprs=" + Arrays.toString(exprs) + ", operandRef=" + operandRef
- + '}';
- }
- }
-
- @Override
- public String toString() {
- return "TruthTable{" + "allOperands=" + allOperands + ", expr=" + expr + '}';
- }
-
- public static class TruthTableBuilder<T> extends ExprBuilder<T> {
-
- public TruthTableBuilder(Comparator<T> operandComparator) {
- super(operandComparator);
- }
-
- public TruthTable build() {
- Expr expr = buildExpr();
- return new TruthTable(new ArrayList<>(allOperandSet), expr);
+ String exprStr = StringUtils.join(operands, ", ", "[", "]");
+ return "Expr{op=" + op + ", operands=" + exprStr + ", operand=" + operand + '}';
}
}
public static class ExprBuilder<T> {
- Set<Expr> allOperandSet = new HashSet<>();
+ Set<Expr<T>> allOperandSet = new HashSet<>();
Deque<Operator> operatorStack = new ArrayDeque<>();
- Deque<List<Expr>> exprsStack = new ArrayDeque<>();
+ Deque<List<Expr<T>>> exprsStack = new ArrayDeque<>();
Comparator<T> operandComparator;
public ExprBuilder(Comparator<T> operandComparator) {
@@ -250,28 +241,27 @@ public class TruthTable {
exprsStack.push(new LinkedList<>());
}
- public ExprBuilder compositeStart(Operator operator) {
+ public ExprBuilder<T> compositeStart(Operator operator) {
operatorStack.push(operator);
exprsStack.push(new LinkedList<>());
return this;
}
- public ExprBuilder compositeEnd() {
- Expr<T> composited = new Expr<>(operatorStack.pop(), exprsStack.pop().toArray(new Expr[0]),
- operandComparator);
- exprsStack.peek().add(composited);
+ public ExprBuilder<T> compositeEnd() {
+ Expr<T> composited = new Expr<>(operatorStack.pop(), operandComparator, exprsStack.pop());
+ Objects.requireNonNull(exprsStack.peek()).add(composited);
return this;
}
- public ExprBuilder addOperand(T operandRef) {
- Expr<T> expr = new Expr<>(operandRef, operandComparator);
+ public ExprBuilder<T> addExpr(T operand) {
+ Expr<T> expr = new Expr<>(operand, operandComparator);
allOperandSet.add(expr);
- exprsStack.peek().add(expr);
+ Objects.requireNonNull(exprsStack.peek()).add(expr);
return this;
}
- protected Expr buildExpr() {
- return exprsStack.peek().get(0);
+ protected Expr<T> build() {
+ return Objects.requireNonNull(exprsStack.peek()).get(0);
}
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/JoinDescNonEquiCompBean.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/JoinDescNonEquiCompBean.java
deleted file mode 100644
index e224ee256f..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/JoinDescNonEquiCompBean.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.metadata.model.tool;
-
-import java.util.Objects;
-
-import org.apache.kylin.metadata.model.JoinDesc;
-
-import lombok.Getter;
-
-public class JoinDescNonEquiCompBean {
-
- @Getter
- private JoinDesc joinDesc;
-
- public JoinDescNonEquiCompBean(JoinDesc joinDesc) {
- this.joinDesc = joinDesc;
- }
-
- @Override
- public int hashCode() {
- return joinDesc.hashCode() * 31 + Objects.hashCode(joinDesc.getNonEquiJoinCondition());
- }
-
- @Override
- public boolean equals(Object o2) {
- if (Objects.isNull(o2)) {
- return false;
- }
-
- JoinDesc joinDesc2 = ((JoinDescNonEquiCompBean) o2).getJoinDesc();
-
- //check equi-join
- if (!Objects.equals(this.joinDesc, joinDesc2)) {
- return false;
- }
-
- //check non-equi-join
- if (!Objects.equals(this.joinDesc.getNonEquiJoinCondition(), joinDesc2.getNonEquiJoinCondition())) {
- return false;
- }
-
- return true;
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparator.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparator.java
deleted file mode 100644
index b60896f994..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparator.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model.tool;
-
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import org.apache.calcite.sql.SqlKind;
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.NonEquiJoinConditionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class NonEquiJoinConditionComparator {
-
- private static Logger logger = LoggerFactory.getLogger(NonEquiJoinConditionComparator.class);
-
- private NonEquiJoinConditionComparator() {
- }
-
- public static boolean equals(NonEquiJoinCondition cond1, NonEquiJoinCondition cond2) {
- try {
- return TruthTable.equals(createTruthTable(cond1), createTruthTable(cond2));
- } catch (Throwable e) {
- logger.error("Error on compareing cond1 {}, cond2 {}", cond1, cond2, e);
- return false;
- }
- }
-
- private static TruthTable createTruthTable(NonEquiJoinCondition nonEquiJoinCondition) {
- TruthTable.TruthTableBuilder<NonEquiJoinCondition> builder = new TruthTable.TruthTableBuilder<>(
- new NonEquiJoinConditionOperandComparator());
- buildExpr(nonEquiJoinCondition, builder);
- return builder.build();
- }
-
- private static Map<SqlKind, TruthTable.Operator> compositeOperatorMapping = new HashMap<>();
- static {
- compositeOperatorMapping.put(SqlKind.AND, TruthTable.Operator.AND);
- compositeOperatorMapping.put(SqlKind.OR, TruthTable.Operator.OR);
- compositeOperatorMapping.put(SqlKind.NOT, TruthTable.Operator.NOT);
- }
-
- private static void buildExpr(NonEquiJoinCondition nonEquiJoinCondition,
- TruthTable.TruthTableBuilder<NonEquiJoinCondition> builder) {
- switch (nonEquiJoinCondition.getOp()) {
- case AND:
- case NOT:
- case OR:
- builder.compositeStart(compositeOperatorMapping.get(nonEquiJoinCondition.getOp()));
- for (int i = 0; i < nonEquiJoinCondition.getOperands().length; i++) {
- buildExpr(nonEquiJoinCondition.getOperands()[i], builder);
- }
- builder.compositeEnd();
- break;
- default:
- addOperand(nonEquiJoinCondition, builder);
- break;
- }
- }
-
- private static void addOperand(NonEquiJoinCondition nonEquiJoinCondition,
- TruthTable.TruthTableBuilder<NonEquiJoinCondition> builder) {
- NonEquiJoinCondition normalized = nonEquiJoinCondition.copy();
- if (inverseCondOperator(normalized)) {
- builder.compositeStart(TruthTable.Operator.NOT);
- normalizedCondOperandOrderings(normalized);
- builder.addOperand(normalized);
- builder.compositeEnd();
- } else {
- normalizedCondOperandOrderings(normalized);
- builder.addOperand(normalized);
- }
- }
-
- private static Map<SqlKind, SqlKind> operatorInverseMapping = new HashMap<>();
- static {
- // NOT_OP -> NOT(OP)
- operatorInverseMapping.put(SqlKind.NOT_EQUALS, SqlKind.EQUALS);
- operatorInverseMapping.put(SqlKind.NOT_IN, SqlKind.IN);
- operatorInverseMapping.put(SqlKind.IS_NOT_NULL, SqlKind.IS_NULL);
- operatorInverseMapping.put(SqlKind.IS_NOT_FALSE, SqlKind.IS_FALSE);
- operatorInverseMapping.put(SqlKind.IS_NOT_TRUE, SqlKind.IS_TRUE);
- operatorInverseMapping.put(SqlKind.IS_NOT_DISTINCT_FROM, SqlKind.IS_DISTINCT_FROM);
- // < -> NOT(>=), <= -> NOT(>)
- operatorInverseMapping.put(SqlKind.LESS_THAN, SqlKind.GREATER_THAN_OR_EQUAL);
- operatorInverseMapping.put(SqlKind.LESS_THAN_OR_EQUAL, SqlKind.GREATER_THAN);
- }
-
- private static boolean inverseCondOperator(NonEquiJoinCondition cond) {
- if (operatorInverseMapping.containsKey(cond.getOp())) {
- cond.setOp(operatorInverseMapping.get(cond.getOp()));
- cond.setOpName(cond.getOp().sql);
- return true;
- }
- return false;
- }
-
- private static void normalizedCondOperandOrderings(NonEquiJoinCondition nonEquiJoinCondition) {
- // if operands ordering does not matter, sort by operands' digest
- // case =, <>
- if (nonEquiJoinCondition.getOp() == SqlKind.EQUALS) {
- Arrays.sort(nonEquiJoinCondition.getOperands(), Comparator.comparing(NonEquiJoinCondition::toString));
- }
-
- // sort IN args
- if (nonEquiJoinCondition.getOp() == SqlKind.IN) {
- Arrays.sort(nonEquiJoinCondition.getOperands(), 1, nonEquiJoinCondition.getOperands().length,
- Comparator.comparing(NonEquiJoinCondition::toString));
- }
- }
-
- public static class NonEquiJoinConditionOperandComparator implements Comparator<NonEquiJoinCondition> {
-
- @Override
- public int compare(NonEquiJoinCondition cond1, NonEquiJoinCondition cond2) {
- if (!(Objects.equals(cond1.getOp(), cond2.getOp())
- && cond1.getOperands().length == cond2.getOperands().length
- && Objects.equals(cond1.getType(), cond2.getType())
- && Objects.equals(cond1.getDataType(), cond2.getDataType()))) {
- return 1;
- }
-
- // compare opName on for SqlKind OTHER
- if (cond1.getOp() == SqlKind.OTHER || cond1.getOp() == SqlKind.OTHER_FUNCTION) {
- if (!Objects.equals(cond1.getOpName(), cond2.getOpName())) {
- return 1;
- }
- }
-
- if (cond1.getType() == NonEquiJoinConditionType.LITERAL) {
- return Objects.equals(cond1.getValue(), cond2.getValue()) ? 0 : 1;
- } else if (cond1.getType() == NonEquiJoinConditionType.COLUMN) {
- return Objects.equals(cond1.getColRef().getColumnDesc(), cond2.getColRef().getColumnDesc()) ? 0 : 1;
- } else {
- for (int i = 0; i < cond1.getOperands().length; i++) {
- if (compare(cond1.getOperands()[i], cond2.getOperands()[i]) != 0) {
- return 1;
- }
- }
- return 0;
- }
- }
-
- }
-
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionVisitor.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionVisitor.java
deleted file mode 100644
index 269bc7182c..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionVisitor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model.tool;
-
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.NonEquiJoinConditionType;
-
-public interface NonEquiJoinConditionVisitor {
-
- default NonEquiJoinCondition visit(NonEquiJoinCondition nonEquiJoinCondition) {
- if (nonEquiJoinCondition == null) {
- return null;
- }
-
- if (nonEquiJoinCondition.getType() == NonEquiJoinConditionType.LITERAL) {
- return visitLiteral(nonEquiJoinCondition);
- } else if (nonEquiJoinCondition.getType() == NonEquiJoinConditionType.COLUMN) {
- return visitColumn(nonEquiJoinCondition);
- } else if (nonEquiJoinCondition.getType() == NonEquiJoinConditionType.EXPRESSION) {
- return visitExpression(nonEquiJoinCondition);
- } else {
- return visit(nonEquiJoinCondition);
- }
- }
-
- default NonEquiJoinCondition visitExpression(NonEquiJoinCondition nonEquiJoinCondition) {
- NonEquiJoinCondition[] ops = new NonEquiJoinCondition[nonEquiJoinCondition.getOperands().length];
- for (int i = 0; i < nonEquiJoinCondition.getOperands().length; i++) {
- ops[i] = visit(nonEquiJoinCondition.getOperands()[i]);
- }
- nonEquiJoinCondition.setOperands(ops);
- return nonEquiJoinCondition;
- }
-
- default NonEquiJoinCondition visitColumn(NonEquiJoinCondition nonEquiJoinCondition) {
- return nonEquiJoinCondition;
- }
-
- default NonEquiJoinCondition visitLiteral(NonEquiJoinCondition nonEquiJoinCondition) {
- return nonEquiJoinCondition;
- }
-
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TypedLiteralConverter.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TypedLiteralConverter.java
deleted file mode 100644
index 4378929a18..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/TypedLiteralConverter.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model.tool;
-
-import java.math.BigDecimal;
-
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.util.DateString;
-import org.apache.calcite.util.TimeString;
-import org.apache.calcite.util.TimestampString;
-import org.apache.kylin.metadata.model.DataType;
-
-public class TypedLiteralConverter {
- private static final String NULL = "KY_LITERAL_NULL";
-
- public static Object stringValueToTypedValue(String value, DataType dataType) {
- if (value.equals(NULL)) {
- return null;
- }
- switch (dataType.getTypeName()) {
- case CHAR:
- case VARCHAR:
- return value;
- case DECIMAL:
- return new BigDecimal(value);
- case BIGINT:
- return Long.parseLong(value);
- case SMALLINT:
- return Short.parseShort(value);
- case TINYINT:
- return Byte.parseByte(value);
- case INTEGER:
- return Integer.parseInt(value);
- case DOUBLE:
- return Double.parseDouble(value);
- case FLOAT:
- case REAL:
- return Float.parseFloat(value);
- case DATE:
- return DateString.fromDaysSinceEpoch(Integer.parseInt(value));
- case TIME:
- case TIME_WITH_LOCAL_TIME_ZONE:
- return TimeString.fromMillisOfDay(Integer.parseInt(value));
- case TIMESTAMP:
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return TimestampString.fromMillisSinceEpoch(Long.parseLong(value));
- case BOOLEAN:
- return Boolean.parseBoolean(value);
- default:
- return value;
- }
- }
-
- public static String typedLiteralToString(RexLiteral literal) {
- if (literal.getValue3() == null) {
- return NULL;
- }
- return String.valueOf(literal.getValue3());
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/JoinDescUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/JoinDescUtil.java
index 22edf67320..3c5bda43e6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/JoinDescUtil.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/JoinDescUtil.java
@@ -28,11 +28,10 @@ import java.util.Objects;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.NDataModel.TableKind;
import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.NonEquiJoinConditionType;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.NDataModel.TableKind;
public class JoinDescUtil {
@@ -98,13 +97,13 @@ public class JoinDescUtil {
private static NonEquiJoinCondition convertNonEquiJoinCondition(NonEquiJoinCondition cond, TableRef pkTblRef,
TableRef fkTblRef) {
- if (cond.getType() == NonEquiJoinConditionType.EXPRESSION) {
+ if (cond.getType() == NonEquiJoinCondition.Type.EXPRESSION) {
return new NonEquiJoinCondition(cond.getOpName(), cond.getOp(),
Arrays.stream(cond.getOperands())
.map(condInput -> convertNonEquiJoinCondition(condInput, pkTblRef, fkTblRef))
.toArray(NonEquiJoinCondition[]::new),
cond.getDataType());
- } else if (cond.getType() == NonEquiJoinConditionType.LITERAL) {
+ } else if (cond.getType() == NonEquiJoinCondition.Type.LITERAL) {
return cond;
} else {
return new NonEquiJoinCondition(convertColumn(cond.getColRef(), pkTblRef, fkTblRef), cond.getDataType());
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2CondChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2CondChecker.java
index fdd7c56481..1f2c218633 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2CondChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2CondChecker.java
@@ -17,17 +17,22 @@
*/
package org.apache.kylin.metadata.model.util.scd2;
-import static org.apache.kylin.metadata.model.NonEquiJoinCondition.SimplifiedNonEquiJoinCondition;
+import static org.apache.kylin.metadata.model.NonEquiJoinCondition.SimplifiedJoinCondition;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import org.apache.calcite.sql.SqlKind;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.NDataModel;
/**
@@ -36,6 +41,10 @@ import org.apache.kylin.metadata.model.NDataModel;
public class SCD2CondChecker {
public static final SCD2CondChecker INSTANCE = new SCD2CondChecker();
+ private static final Set<Set<SqlKind>> SCD_2_JOIN_PAIRS = ImmutableSet.of(
+ Sets.newHashSet(SqlKind.GREATER_THAN, SqlKind.LESS_THAN_OR_EQUAL),
+ Sets.newHashSet(SqlKind.GREATER_THAN_OR_EQUAL, SqlKind.LESS_THAN_OR_EQUAL),
+ Sets.newHashSet(SqlKind.GREATER_THAN_OR_EQUAL, SqlKind.LESS_THAN));
/**
* SCD2 must have equi condition
@@ -45,22 +54,21 @@ public class SCD2CondChecker {
}
public boolean isScd2Model(NDataModel nDataModel) {
- if (nDataModel.getJoinTables().stream().filter(joinTableDesc -> joinTableDesc.getJoin().isNonEquiJoin())
- .count() == 0) {
+ if (nDataModel.getJoinTables().stream().noneMatch(joinTableDesc -> joinTableDesc.getJoin().isNonEquiJoin())) {
return false;
}
- return nDataModel.getJoinTables().stream().filter(joinTableDesc -> joinTableDesc.getJoin().isNonEquiJoin())
- .allMatch(joinTableDesc -> SCD2NonEquiCondSimplification.INSTANCE
- .simplifiedSCD2CondConvertChecker(joinTableDesc.getJoin()));
+ return nDataModel.getJoinTables().stream() //
+ .filter(joinTableDesc -> joinTableDesc.getJoin().isNonEquiJoin())
+ .allMatch(joinTableDesc -> Scd2Simplifier.INSTANCE.isValidScd2JoinDesc(joinTableDesc.getJoin()));
}
public boolean checkFkPkPairUnique(SimplifiedJoinDesc joinDesc) {
- return checkFkPkPairUnique(SCD2NonEquiCondSimplification.INSTANCE.simplifyFksPks(joinDesc.getForeignKey(),
- joinDesc.getPrimaryKey()), joinDesc.getSimplifiedNonEquiJoinConditions());
+ return checkFkPkPairUnique(Scd2Simplifier.INSTANCE.buildAndCheckEqualConditions(joinDesc),
+ joinDesc.getSimplifiedNonEquiJoinConditions());
}
- public boolean checkSCD2NonEquiJoinCondPair(final List<SimplifiedNonEquiJoinCondition> simplified) {
+ public boolean checkSCD2NonEquiJoinCondPair(final List<SimplifiedJoinCondition> simplified) {
if (CollectionUtils.isEmpty(simplified)) {
return false;
}
@@ -71,61 +79,75 @@ public class SCD2CondChecker {
return false;
}
- Map<String, Integer> mappingCount = new HashMap<>();
-
- for (SimplifiedNonEquiJoinCondition cond : simplified) {
+ Map<String, List<SqlKind>> mapping = Maps.newHashMap();
+ for (SimplifiedJoinCondition cond : simplified) {
if (!checkSCD2SqlOp(cond.getOp())) {
return false;
}
- int inrc = cond.getOp() == SqlKind.GREATER_THAN_OR_EQUAL ? 1 : -1;
+ mapping.putIfAbsent(cond.getForeignKey(), Lists.newArrayList());
+ mapping.putIfAbsent(cond.getPrimaryKey(), Lists.newArrayList());
+ mapping.get(cond.getForeignKey()).add(cond.getOp());
+ mapping.get(cond.getPrimaryKey()).add(inverse(cond.getOp()));
+ }
+ mapping.entrySet().removeIf(entry -> entry.getValue().size() != 2);
- mappingCount.put(cond.getForeignKey(), mappingCount.getOrDefault(cond.getForeignKey(), 0) + inrc);
+ return mapping.values().stream().allMatch(this::isValidOperatorPair);
+ }
+ public static SqlKind inverse(SqlKind kind) {
+ switch (kind) {
+ case GREATER_THAN:
+ return SqlKind.LESS_THAN;
+ case LESS_THAN:
+ return SqlKind.GREATER_THAN;
+ case GREATER_THAN_OR_EQUAL:
+ return SqlKind.LESS_THAN_OR_EQUAL;
+ case LESS_THAN_OR_EQUAL:
+ return SqlKind.GREATER_THAN_OR_EQUAL;
+ default:
+ return null;
}
+ }
- return !mappingCount.values().stream().anyMatch(count -> count != 0);
-
+ private boolean isValidOperatorPair(List<SqlKind> ops) {
+ if (ops.size() != 2 || ops.get(0) == ops.get(1)) {
+ return false;
+ }
+ return SCD_2_JOIN_PAIRS.contains(Sets.newHashSet(ops));
}
- boolean checkFkPkPairUnique(List<SimplifiedNonEquiJoinCondition> equiFkPks,
- List<SimplifiedNonEquiJoinCondition> nonEquiFkPks) {
- List<SimplifiedNonEquiJoinCondition> allFkPks = new ArrayList<>();
+ boolean checkFkPkPairUnique(List<SimplifiedJoinCondition> equiFkPks, List<SimplifiedJoinCondition> nonEquiFkPks) {
+ List<SimplifiedJoinCondition> allFkPks = new ArrayList<>();
allFkPks.addAll(equiFkPks);
allFkPks.addAll(nonEquiFkPks);
- HashSet pairSet = new HashSet<String>();
-
- for (int i = 0; i < allFkPks.size(); i++) {
- String key = allFkPks.get(i).getForeignKey() + allFkPks.get(i).getPrimaryKey();
+ Set<String> pairSet = new HashSet<>();
+ for (SimplifiedJoinCondition allFkPk : allFkPks) {
+ String[] joinKeys = new String[] { allFkPk.getPrimaryKey(), allFkPk.getForeignKey() };
+ Arrays.sort(joinKeys);
+ String key = String.join(",", joinKeys);
if (pairSet.contains(key)) {
return false;
}
pairSet.add(key);
-
}
return true;
-
}
- /**
- * SCD2 only support =, >=, <
- * @param op
- * @return
- */
private boolean checkSCD2SqlOp(SqlKind op) {
-
if (Objects.isNull(op)) {
return false;
}
switch (op) {
case GREATER_THAN_OR_EQUAL:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN:
case LESS_THAN:
return true;
default:
return false;
}
-
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java
deleted file mode 100644
index d77005533d..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2Exception.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.metadata.model.util.scd2;
-
-public class SCD2Exception extends RuntimeException {
-
- public SCD2Exception(String message) {
- super(message);
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2NonEquiCondSimplification.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2NonEquiCondSimplification.java
deleted file mode 100644
index 19767de05b..0000000000
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2NonEquiCondSimplification.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.metadata.model.util.scd2;
-
-import static org.apache.kylin.metadata.model.NonEquiJoinCondition.SimplifiedNonEquiJoinCondition;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import org.apache.calcite.sql.SqlKind;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.NonEquiJoinConditionType;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-/**
- * simplify non-equi-join
- */
-public class SCD2NonEquiCondSimplification {
-
- public static final SCD2NonEquiCondSimplification INSTANCE = new SCD2NonEquiCondSimplification();
-
- /**
- * convert cond from origin model.
- * it may contain =,>=,< and so on
- *
- * if the condition is not SCD2, return null
- * @param joinDesc
- * @return
- */
- public SimplifiedJoinDesc convertToSimplifiedSCD2Cond(@Nullable JoinDesc joinDesc) throws SCD2Exception {
-
- if (Objects.isNull(joinDesc)) {
- return null;
- }
-
- NonEquiJoinCondition nonEquiJoinCondition = joinDesc.getNonEquiJoinCondition();
- //null, or is not `and` cond
- if (Objects.isNull(nonEquiJoinCondition) || nonEquiJoinCondition.getOp() != SqlKind.AND) {
- throw new SCD2Exception("scd2 must has non-equi cond and only support `AND` expression ");
- }
-
- SimplifiedJoinDesc convertedJoinDesc = new SimplifiedJoinDesc();
-
- List<SimplifiedNonEquiJoinCondition> nonEquiJoinConds = Lists.newArrayList();
- List<SimplifiedNonEquiJoinCondition> equiJoinConds = simplifyFksPks(joinDesc.getForeignKey(),
- joinDesc.getPrimaryKey());
-
- NonEquiJoinCondition[] nonEquiJoinConditions = nonEquiJoinCondition.getOperands();
-
- for (NonEquiJoinCondition nonEquivCond : nonEquiJoinConditions) {
- SimplifiedNonEquiJoinCondition scd2Cond = simplifySCD2ChildCond(nonEquivCond);
-
- //any child is illegal, return null
- if (Objects.isNull(scd2Cond)) {
- throw new SCD2Exception("it has illegal scd2 child expression ");
- }
- nonEquiJoinConds.add(scd2Cond);
- }
-
- //check >=,< pair match
- //check = join , fk and pk
-
- if (!SCD2CondChecker.INSTANCE.checkSCD2NonEquiJoinCondPair(nonEquiJoinConds)) {
- throw new SCD2Exception("the `>=` and `<` must be pair");
- }
-
- if (CollectionUtils.isEmpty(equiJoinConds)) {
- throw new SCD2Exception("scd2 should have `=` at leas one");
- }
-
- if (CollectionUtils.isEmpty(nonEquiJoinConds)) {
- throw new SCD2Exception("scd2 should have non-equi condition at leas one");
- }
-
- //check unique
- SCD2CondChecker.INSTANCE.checkFkPkPairUnique(equiJoinConds, nonEquiJoinConds);
-
- convertedJoinDesc.setSimplifiedNonEquiJoinConditions(nonEquiJoinConds);
-
- //extract fk\pk from non-equi cond
- simplifyFksPks(equiJoinConds, convertedJoinDesc);
-
- return convertedJoinDesc;
-
- }
-
- List<SimplifiedNonEquiJoinCondition> simplifyFksPks(String[] fks, String[] pks) {
-
- List<SimplifiedNonEquiJoinCondition> simplifiedFksPks = new ArrayList<>();
-
- for (int i = 0; i < fks.length; i++) {
- simplifiedFksPks.add(new SimplifiedNonEquiJoinCondition(fks[i], pks[i], SqlKind.EQUALS));
- }
-
- return simplifiedFksPks;
-
- }
-
- public TblColRef[] extractFksFromNonEquiJoinDesc(@Nonnull JoinDesc joinDesc) {
- Preconditions.checkNotNull(joinDesc, "joinDesc is null");
-
- List<TblColRef> fkList = convertToSimplifiedSCD2Cond(joinDesc).getSimplifiedNonEquiJoinConditions().stream()
- .map(SimplifiedNonEquiJoinCondition::getFk).distinct().collect(Collectors.toList());
-
- TblColRef[] fks = new TblColRef[fkList.size()];
- fkList.toArray(fks);
- return fks;
- }
-
- boolean simplifiedSCD2CondConvertChecker(@Nullable JoinDesc joinDesc) {
- try {
- return convertToSimplifiedSCD2Cond(joinDesc) != null;
- } catch (Exception e) {
- return false;
- }
- }
-
- /**
- * non-equi join condition to fks,pks
- * @param equiJoinConds
- * @param joinDesc
- */
- private void simplifyFksPks(List<SimplifiedNonEquiJoinCondition> equiJoinConds, JoinDesc joinDesc) {
-
- int len = equiJoinConds.size();
-
- String[] fks = new String[len];
- String[] pks = new String[len];
-
- for (int i = 0; i < len; i++) {
- fks[i] = equiJoinConds.get(i).getForeignKey();
- pks[i] = equiJoinConds.get(i).getPrimaryKey();
- }
-
- joinDesc.setForeignKey(fks);
- joinDesc.setPrimaryKey(pks);
-
- }
-
- /**
- * simplify child cond
- * (a>=b) and (a<c) ->
- * {"a","b",">="} {"a","c","<"}
- * @param nonEquiJoinCondition
- * @return
- */
- private SimplifiedNonEquiJoinCondition simplifySCD2ChildCond(NonEquiJoinCondition nonEquiJoinCondition) {
- if (nonEquiJoinCondition.getType() != NonEquiJoinConditionType.EXPRESSION
- || nonEquiJoinCondition.getOperands().length != 2) {
- return null;
- }
-
- List<Pair<String, TblColRef>> fkPk = Arrays.stream(nonEquiJoinCondition.getOperands())
- .map(nonEquiJoinConditionChild -> {
- if (nonEquiJoinConditionChild.getOp() == SqlKind.CAST
- && nonEquiJoinConditionChild.getOperands().length == 1) {
- return nonEquiJoinConditionChild.getOperands()[0];
- } else {
- return nonEquiJoinConditionChild;
- }
- }).collect(Collectors.toList()).stream()
- .filter(nonEquiJoinConditionChild -> nonEquiJoinConditionChild.getOperands().length == 0
- || nonEquiJoinConditionChild.getType() == NonEquiJoinConditionType.COLUMN)
- .map(nonEquiJoinCondition1 -> new Pair<String, TblColRef>(nonEquiJoinCondition1.getValue(),
- nonEquiJoinCondition1.getColRef()))
- .collect(Collectors.toList());
- if (CollectionUtils.isEmpty(fkPk) || fkPk.size() != 2) {
- return null;
- }
-
- return new SimplifiedNonEquiJoinCondition(fkPk.get(0).getFirst(), fkPk.get(0).getSecond(),
- fkPk.get(1).getFirst(), fkPk.get(1).getSecond(), nonEquiJoinCondition.getOp());
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2SqlConverter.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2SqlConverter.java
index 24a2b7f283..b2448f085d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2SqlConverter.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SCD2SqlConverter.java
@@ -42,7 +42,7 @@ public class SCD2SqlConverter {
* @return
*/
public String genSCD2SqlStr(JoinDesc joinDesc,
- List<NonEquiJoinCondition.SimplifiedNonEquiJoinCondition> simplifiedNonEquiJoinConditions) {
+ List<NonEquiJoinCondition.SimplifiedJoinCondition> simplifiedNonEquiJoinConditions) {
StringBuilder sb = new StringBuilder();
sb.append("select * from ").append(toJoinDescQuotedString(joinDesc))
@@ -65,7 +65,7 @@ public class SCD2SqlConverter {
return quotedIdentifierStr(cols[0]) + "." + quotedIdentifierStr(cols[1]);
}
- private String genNonEquiWithSimplified(List<NonEquiJoinCondition.SimplifiedNonEquiJoinCondition> simplified) {
+ private String genNonEquiWithSimplified(List<NonEquiJoinCondition.SimplifiedJoinCondition> simplified) {
return simplified.stream()
.map(simplifiedNonEquiJoinCondition -> "("
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/Scd2Simplifier.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/Scd2Simplifier.java
new file mode 100644
index 0000000000..f424684dbd
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/Scd2Simplifier.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.metadata.model.util.scd2;
+
+import static org.apache.kylin.metadata.model.NonEquiJoinCondition.SimplifiedJoinCondition;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.calcite.sql.SqlKind;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.NonEquiJoinCondition;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * simplify non-equi-join
+ */
+public class Scd2Simplifier {
+
+ public static final Scd2Simplifier INSTANCE = new Scd2Simplifier();
+
+ /**
+ * convert cond from origin model.
+ * it may contain =,>=,< and so on
+ *
+ * if the condition is not SCD2, return null
+ * @param joinDesc
+ * @return
+ */
+ public SimplifiedJoinDesc simplifyScd2Conditions(@Nullable JoinDesc joinDesc) {
+ if (Objects.isNull(joinDesc)) {
+ return null;
+ }
+
+ List<SimplifiedJoinCondition> equalJoinPairs = buildAndCheckEqualConditions(joinDesc);
+ List<SimplifiedJoinCondition> nonEqualJoinPairs = buildAndCheckNeqConditions(joinDesc);
+
+ // check all the join-condition paris are unique
+ SCD2CondChecker.INSTANCE.checkFkPkPairUnique(equalJoinPairs, nonEqualJoinPairs);
+ SimplifiedJoinDesc convertedJoinDesc = new SimplifiedJoinDesc();
+ convertedJoinDesc.setSimplifiedNonEquiJoinConditions(nonEqualJoinPairs);
+ convertedJoinDesc.simplifyEqualJoinPairs(equalJoinPairs);
+ return convertedJoinDesc;
+ }
+
+ private List<SimplifiedJoinCondition> buildAndCheckNeqConditions(JoinDesc joinDesc) {
+ NonEquiJoinCondition nonEquiJoinCondition = joinDesc.getNonEquiJoinCondition();
+ //null, or is not `and` cond
+ if (Objects.isNull(nonEquiJoinCondition) || nonEquiJoinCondition.getOp() != SqlKind.AND) {
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_CAN_ONLY_CONNECT_BY_AND);
+ }
+ List<SimplifiedJoinCondition> nonEqualJoinPairs = Lists.newArrayList();
+ for (NonEquiJoinCondition nonEquivCond : nonEquiJoinCondition.getOperands()) {
+ SimplifiedJoinCondition scd2Cond = simplifySCD2ChildCond(nonEquivCond, joinDesc);
+ if (Objects.isNull(scd2Cond)) {
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_CONTAINS_ILLEGAL_EXPRESSIONS);
+ }
+ nonEqualJoinPairs.add(scd2Cond);
+ }
+
+ if (!SCD2CondChecker.INSTANCE.checkSCD2NonEquiJoinCondPair(nonEqualJoinPairs)) {
+ throw new KylinException(ErrorCodeServer.SCD2_CONDITION_MUST_APPEAR_IN_PAIRS);
+ }
+
+ if (CollectionUtils.isEmpty(nonEqualJoinPairs)) {
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_REQUIRES_AT_LEAST_ONE_NON_EQUAL_CONDITION);
+ }
+ return nonEqualJoinPairs;
+ }
+
+ List<SimplifiedJoinCondition> buildAndCheckEqualConditions(JoinDesc joinDesc) {
+ String[] fks = joinDesc.getForeignKey();
+ String[] pks = joinDesc.getPrimaryKey();
+ List<SimplifiedJoinCondition> simplifiedFksPks = new ArrayList<>();
+ for (int i = 0; i < fks.length; i++) {
+ simplifiedFksPks.add(new SimplifiedJoinCondition(fks[i], pks[i], SqlKind.EQUALS));
+ }
+ if (CollectionUtils.isEmpty(simplifiedFksPks)) {
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_REQUIRES_AT_LEAST_ONE_EQUAL_CONDITION);
+ }
+
+ return simplifiedFksPks;
+ }
+
+ public TblColRef[] extractNeqFks(@Nonnull JoinDesc joinDesc) {
+ Preconditions.checkNotNull(joinDesc, "joinDesc is null");
+
+ List<TblColRef> fkList = simplifyScd2Conditions(joinDesc).getSimplifiedNonEquiJoinConditions().stream()
+ .map(SimplifiedJoinCondition::getFk).distinct().collect(Collectors.toList());
+
+ TblColRef[] fks = new TblColRef[fkList.size()];
+ fkList.toArray(fks);
+ return fks;
+ }
+
+ boolean isValidScd2JoinDesc(@Nullable JoinDesc joinDesc) {
+ try {
+ return simplifyScd2Conditions(joinDesc) != null;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ /**
+ * Simplify child condition, for example:
+ * {@code (a>=b) and (a<c) } will be transformed to {@code {"a","b",">="}, {"a","c","<"}}
+ */
+ public SimplifiedJoinCondition simplifySCD2ChildCond(NonEquiJoinCondition neqCondition, JoinDesc joinDesc) {
+ if (neqCondition.getType() != NonEquiJoinCondition.Type.EXPRESSION || neqCondition.getOperands().length != 2) {
+ return null;
+ }
+
+ List<Pair<String, TblColRef>> scd2Pairs = Arrays.stream(neqCondition.getOperands())
+ .map(nonEquiJoinConditionChild -> {
+ if (nonEquiJoinConditionChild.getOp() == SqlKind.CAST
+ && nonEquiJoinConditionChild.getOperands().length == 1) {
+ return nonEquiJoinConditionChild.getOperands()[0];
+ } else {
+ return nonEquiJoinConditionChild;
+ }
+ })
+ .filter(nonEquiJoinConditionChild -> nonEquiJoinConditionChild.getOperands().length == 0
+ || nonEquiJoinConditionChild.getType() == NonEquiJoinCondition.Type.COLUMN)
+ .map(nonEquiJoinCondition1 -> new Pair<>(nonEquiJoinCondition1.getValue(),
+ nonEquiJoinCondition1.getColRef()))
+ .collect(Collectors.toList());
+ if (CollectionUtils.isEmpty(scd2Pairs) || scd2Pairs.size() != 2) {
+ return null;
+ }
+
+ // pk-fk should be consistent with the join table, just for display
+ Pair<String, TblColRef> left = scd2Pairs.get(0);
+ Pair<String, TblColRef> right = scd2Pairs.get(1);
+ String[] fks = joinDesc.getForeignKey();
+ if (fks.length == 0) {
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_REQUIRES_AT_LEAST_ONE_NON_EQUAL_CONDITION);
+ }
+ String fkTableAlias = StringUtils.split(fks[0], '.')[0];
+ String fkTableAliasOfNeqCondition = StringUtils.split(left.getFirst(), '.')[0];
+ return fkTableAliasOfNeqCondition.equalsIgnoreCase(fkTableAlias)
+ ? new SimplifiedJoinCondition(left.getKey(), left.getValue(), right.getKey(), right.getValue(),
+ neqCondition.getOp())
+ : new SimplifiedJoinCondition(right.getKey(), right.getValue(), left.getKey(), left.getValue(),
+ SCD2CondChecker.inverse(neqCondition.getOp()));
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SimplifiedJoinDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SimplifiedJoinDesc.java
index 1fe9e54dc2..31999ac38b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SimplifiedJoinDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/util/scd2/SimplifiedJoinDesc.java
@@ -29,11 +29,30 @@ import lombok.Setter;
@Getter
@Setter
-public class SimplifiedJoinDesc extends JoinDesc {
+public class SimplifiedJoinDesc extends JoinDesc.NonEquivJoinDesc {
private static final long serialVersionUID = 3422512377209976139L;
@JsonProperty("simplified_non_equi_join_conditions")
- private List<NonEquiJoinCondition.SimplifiedNonEquiJoinCondition> simplifiedNonEquiJoinConditions;
+ private List<NonEquiJoinCondition.SimplifiedJoinCondition> simplifiedNonEquiJoinConditions;
+
+ /**
+ * non-equi join condition to fks,pks
+ */
+ public void simplifyEqualJoinPairs(List<NonEquiJoinCondition.SimplifiedJoinCondition> simplifiedNeqConditions) {
+
+ int len = simplifiedNeqConditions.size();
+
+ String[] fks = new String[len];
+ String[] pks = new String[len];
+
+ for (int i = 0; i < len; i++) {
+ fks[i] = simplifiedNeqConditions.get(i).getForeignKey();
+ pks[i] = simplifiedNeqConditions.get(i).getPrimaryKey();
+ }
+
+ this.setForeignKey(fks);
+ this.setPrimaryKey(pks);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
index 7eb5569f09..a10b271b01 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
@@ -249,8 +249,12 @@ public class NProjectManager {
}
public static KylinConfig getProjectConfig(String project) {
- NProjectManager projectManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
- ProjectInstance projectInstance = projectManager.getProject(project);
- return projectInstance.getConfig();
+ KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+ if (project == null) {
+ logger.warn("The project name is null, return KylinCong.getInstanceFromEnv() instead.");
+ return envConfig;
+ }
+ ProjectInstance projectInstance = NProjectManager.getInstance(envConfig).getProject(project);
+ return projectInstance == null ? envConfig : projectInstance.getConfig();
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
index df4eb94589..95e4704775 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
@@ -34,15 +34,14 @@ public class QueryInterruptChecker {
}
/**
- * @deprecated Use {@link this#checkQueryCanceledOrThreadInterrupted(String, String)} instead.
+ * At most cases, please use {@link this#checkQueryCanceledOrThreadInterrupted(String, String)} instead.
* The semantic of this method is confused in some scenarios.
*/
- @Deprecated
public static void checkThreadInterrupted(String errorMsgLog, String stepInfo) {
if (Thread.currentThread().isInterrupted()) {
log.error("{} {}", QueryContext.current().getQueryId(), errorMsgLog);
if (SlowQueryDetector.getRunningQueries().containsKey(Thread.currentThread())
- && SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser()) {
+ && SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser()) {
throw new UserStopQueryException("");
}
@@ -67,31 +66,31 @@ public class QueryInterruptChecker {
*/
public static void checkQueryCanceledOrThreadInterrupted(String cause, String step) throws InterruptedException {
SlowQueryDetector.QueryEntry entry = SlowQueryDetector.getRunningQueries().getOrDefault(Thread.currentThread(),
- null);
+ null);
if (entry != null) {
if (entry.isStopByUser() && entry.getPlannerCancelFlag().isCancelRequested()
- && Thread.currentThread().isInterrupted()) {
+ && Thread.currentThread().isInterrupted()) {
throw new UserStopQueryException(String.format("Manually stop the query %s. Caused: %s. Step: %s",
- entry.getQueryId(), cause, step));
+ entry.getQueryId(), cause, step));
}
if (entry.getPlannerCancelFlag().isCancelRequested() && Thread.currentThread().isInterrupted()) {
QueryContext.current().getQueryTagInfo().setTimeout(true);
throw new KylinTimeoutException(String.format("Run out of time of the query %s. Caused: %s. Step: %s",
- entry.getQueryId(), cause, step));
+ entry.getQueryId(), cause, step));
}
if (entry.isStopByUser() || entry.getPlannerCancelFlag().isCancelRequested()) {
throw new UserStopQueryException(String.format(
- "You are trying to cancel the query %s with inconsistent states:"
- + " [isStopByUser=%s, isCancelRequested=%s]! Caused: %s. Step: %s",
- entry.getQueryId(), entry.isStopByUser(), entry.getPlannerCancelFlag().isCancelRequested(),
- cause, step));
+ "You are trying to cancel the query %s with inconsistent states:"
+ + " [isStopByUser=%s, isCancelRequested=%s]! Caused: %s. Step: %s",
+ entry.getQueryId(), entry.isStopByUser(), entry.getPlannerCancelFlag().isCancelRequested(),
+ cause, step));
}
}
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedException(String.format("Interrupted on thread %s. Caused: %s. Step: %s",
- Thread.currentThread().getName(), cause, step));
+ Thread.currentThread().getName(), cause, step));
}
}
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/ModelNonEquiCondMock.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/ModelNonEquiCondMock.java
index 774ed11ba6..dbfa75eb5d 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/ModelNonEquiCondMock.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/ModelNonEquiCondMock.java
@@ -22,16 +22,10 @@ import java.util.HashMap;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableRef;
-import org.apache.kylin.metadata.model.TblColRef;
public class ModelNonEquiCondMock {
- private HashMap<String, TableRef> tableRefCache = new HashMap<>();
+ private final HashMap<String, TableRef> tableRefCache = new HashMap<>();
public void clearTableRefCache() {
tableRefCache.clear();
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NeqEquiJoinConditionTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NeqEquiJoinConditionTest.java
new file mode 100644
index 0000000000..f3099526ba
--- /dev/null
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NeqEquiJoinConditionTest.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model;
+
+import java.math.BigDecimal;
+import java.util.function.Function;
+
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.TimestampString;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.NonEquiJoinCondition.TypedLiteralConverter;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class NeqEquiJoinConditionTest {
+
+ ModelNonEquiCondMock nonEquiMock = new ModelNonEquiCondMock();
+
+ @AfterEach
+ public void teardown() {
+ nonEquiMock.clearTableRefCache();
+ }
+
+ @Test
+ void testSimpleEqual() {
+ NonEquiJoinCondition cond1 = nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1");
+ Assertions.assertNotNull(cond1);
+ }
+
+ @Test
+ void testNestedCondEqual() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
+ ModelNonEquiCondMock.composite(SqlKind.AND,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2"),
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a5", "B.b6")),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"), nonEquiMock.colConstantCompareCond(
+ SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR, "SOME TEXT", SqlTypeName.CHAR));
+ Assertions.assertNotNull(cond1);
+ }
+
+ @Test
+ void testOrderingIrrelevantCondEqual() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"),
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"));
+ Assertions.assertEquals(cond1, cond2);
+ }
+
+ @Test
+ void testComparingWithNull() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"));
+ Assertions.assertNotNull(cond1);
+ }
+
+ @Test
+ void testNotEqual() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
+ ModelNonEquiCondMock.composite(SqlKind.AND,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2"),
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a5", "B.b6")),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"), nonEquiMock.colConstantCompareCond(
+ SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR, "SOME TEXT", SqlTypeName.CHAR));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
+ ModelNonEquiCondMock.composite(SqlKind.OR,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b8"),
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a5", "B.b6")),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"), nonEquiMock.colConstantCompareCond(
+ SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR, "SOME TEXT", SqlTypeName.CHAR));
+ Assertions.assertNotEquals(cond1, cond2);
+ }
+
+ @Test
+ void testNotEqualOnTable() {
+ NonEquiJoinCondition cond1 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "B.b1");
+ NonEquiJoinCondition cond2 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "C.b1");
+ Assertions.assertNotEquals(cond1, cond2);
+ }
+
+ @Test
+ void testNotEqualOnColumn() {
+ NonEquiJoinCondition cond1 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "B.b1");
+ NonEquiJoinCondition cond2 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "B.b2");
+ Assertions.assertNotEquals(cond1, cond2);
+ }
+
+ @Test
+ void testNotEqualOnConstant() {
+ NonEquiJoinCondition cond1 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
+ "SOME TEXT", SqlTypeName.CHAR);
+ NonEquiJoinCondition cond2 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
+ "SOME TEXT1", SqlTypeName.CHAR);
+ Assertions.assertNotEquals(cond1, cond2);
+ }
+
+ @Test
+ void testNotEqualOnType() {
+ NonEquiJoinCondition cond1 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
+ "SOME TEXT", SqlTypeName.CHAR);
+ NonEquiJoinCondition cond2 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
+ "SOME TEXT", SqlTypeName.INTEGER);
+ Assertions.assertNotEquals(cond1, cond2);
+ }
+
+ @Test
+ void testCondNormalizations() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.AND,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a8", "B.b8"),
+ ModelNonEquiCondMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a9", "B.b9")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT, nonEquiMock.colOp(SqlKind.IS_NULL, "A.a9")),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN_OR_EQUAL, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a4", "B.b4"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.AND,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "B.b8", "A.a8"),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "B.b9", "A.a9"),
+ nonEquiMock.colOp(SqlKind.IS_NOT_NULL, "A.a9"),
+ ModelNonEquiCondMock.composite(SqlKind.NOT,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN_OR_EQUAL, "A.a4", "B.b4")));
+ Assertions.assertEquals(cond1, cond2);
+ }
+
+ @Test
+ void testCondNormalizationsOnSqlKindOTHER() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"));
+ Assertions.assertEquals(cond1, cond2);
+ }
+
+ @Test
+ void testCondNormalizationsOnSqlKindOTHERNotEqual() {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
+ nonEquiMock.colOp("cos", SqlKind.OTHER_FUNCTION, "A.a9"));
+ Assertions.assertNotEquals(cond1, cond2);
+ }
+
+ @Test
+ void testNonEquJoinDescComparator() {
+
+ Function<NonEquiJoinCondition, JoinDesc> supplierSimpleJoinDesc = (nonEquiJoinCondition) -> {
+ JoinDesc joinDesc = new JoinDesc.NonEquivJoinDesc();
+ joinDesc.setNonEquiJoinCondition(nonEquiJoinCondition);
+ joinDesc.setPrimaryKey(new String[] {});
+ joinDesc.setForeignKey(new String[] {});
+ joinDesc.setForeignKeyColumns(new TblColRef[] {});
+ joinDesc.setPrimaryKeyColumns(new TblColRef[] {});
+ joinDesc.setType("LEFT");
+ return joinDesc;
+ };
+ //nonEqual
+ {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
+ nonEquiMock.colOp("cos", SqlKind.OTHER_FUNCTION, "A.a9"));
+ Assertions.assertNotEquals(supplierSimpleJoinDesc.apply(cond1), supplierSimpleJoinDesc.apply(cond2));
+ }
+
+ //equal
+ {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.EQUALS,
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
+ nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"));
+ Assertions.assertEquals(supplierSimpleJoinDesc.apply(cond1), supplierSimpleJoinDesc.apply(cond2));
+ }
+ //equal
+ {
+ NonEquiJoinCondition cond1 = ModelNonEquiCondMock.composite(SqlKind.AND,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a8", "B.b8"),
+ ModelNonEquiCondMock.composite(SqlKind.NOT,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a9", "B.b9")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT, nonEquiMock.colOp(SqlKind.IS_NULL, "A.a9")),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN_OR_EQUAL, "A.a1", "B.b1"),
+ nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a4", "B.b4"));
+ NonEquiJoinCondition cond2 = ModelNonEquiCondMock.composite(SqlKind.AND,
+ nonEquiMock.colCompareCond(SqlKind.EQUALS, "B.b8", "A.a8"),
+ nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "B.b9", "A.a9"),
+ nonEquiMock.colOp(SqlKind.IS_NOT_NULL, "A.a9"),
+ ModelNonEquiCondMock.composite(SqlKind.NOT,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1")),
+ ModelNonEquiCondMock.composite(SqlKind.NOT,
+ nonEquiMock.colCompareCond(SqlKind.GREATER_THAN_OR_EQUAL, "A.a4", "B.b4")));
+ Assertions.assertEquals(supplierSimpleJoinDesc.apply(cond1), supplierSimpleJoinDesc.apply(cond2));
+ }
+ }
+
+ @Test
+ void testTypedLiteralConverter() {
+ DataType anyType = ModelNonEquiCondMock.mockDataType(SqlTypeName.ANY);
+ Assertions.assertNull(TypedLiteralConverter.stringValueToTypedValue(TypedLiteralConverter.NULL, anyType));
+
+ DataType decimalType = ModelNonEquiCondMock.mockDataType(SqlTypeName.DECIMAL);
+ BigDecimal decimal = new BigDecimal("1.5");
+ Assertions.assertEquals(decimal, TypedLiteralConverter.stringValueToTypedValue("1.5", decimalType));
+
+ DataType longType = ModelNonEquiCondMock.mockDataType(SqlTypeName.BIGINT);
+ Assertions.assertEquals(1L, TypedLiteralConverter.stringValueToTypedValue("1", longType));
+
+ DataType sType = ModelNonEquiCondMock.mockDataType(SqlTypeName.SMALLINT);
+ Assertions.assertEquals((short) 1, TypedLiteralConverter.stringValueToTypedValue("1", sType));
+
+ DataType tinyInt = ModelNonEquiCondMock.mockDataType(SqlTypeName.TINYINT);
+ Assertions.assertEquals((byte) 1, TypedLiteralConverter.stringValueToTypedValue("1", tinyInt));
+
+ DataType intType = ModelNonEquiCondMock.mockDataType(SqlTypeName.INTEGER);
+ Assertions.assertEquals(1, TypedLiteralConverter.stringValueToTypedValue("1", intType));
+
+ DataType dType = ModelNonEquiCondMock.mockDataType(SqlTypeName.DOUBLE);
+ Assertions.assertEquals(1.0d, TypedLiteralConverter.stringValueToTypedValue("1.0", dType));
+
+ DataType fType = ModelNonEquiCondMock.mockDataType(SqlTypeName.FLOAT);
+ Assertions.assertEquals(1.0f, TypedLiteralConverter.stringValueToTypedValue("1", fType));
+
+ DataType realType = ModelNonEquiCondMock.mockDataType(SqlTypeName.REAL);
+ Assertions.assertEquals(1.0f, TypedLiteralConverter.stringValueToTypedValue("1", realType));
+
+ DataType boolType = ModelNonEquiCondMock.mockDataType(SqlTypeName.BOOLEAN);
+ Assertions.assertEquals(false, TypedLiteralConverter.stringValueToTypedValue("false", boolType));
+
+ DataType charType = ModelNonEquiCondMock.mockDataType(SqlTypeName.CHAR);
+ Assertions.assertEquals("a", TypedLiteralConverter.stringValueToTypedValue("a", charType));
+
+ DataType varcharType = ModelNonEquiCondMock.mockDataType(SqlTypeName.VARCHAR);
+ Assertions.assertEquals("abc", TypedLiteralConverter.stringValueToTypedValue("abc", varcharType));
+
+ // why timezone problem?
+ String tsValue = "2023-01-03 12:13:14";
+ long ts = DateFormat.stringToMillis(tsValue);
+ TimestampString expTs = new TimestampString(2023, 1, 3, 4, 13, 14);
+ DataType tsType = ModelNonEquiCondMock.mockDataType(SqlTypeName.TIMESTAMP);
+ Assertions.assertEquals(expTs, TypedLiteralConverter.stringValueToTypedValue(String.valueOf(ts), tsType));
+ }
+}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparatorTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparatorTest.java
deleted file mode 100644
index 9072cbe00d..0000000000
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/tool/NonEquiJoinConditionComparatorTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.model.tool;
-
-import java.util.function.Function;
-
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.NonEquiJoinCondition;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.ModelNonEquiCondMock;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class NonEquiJoinConditionComparatorTest {
-
- ModelNonEquiCondMock nonEquiMock = new ModelNonEquiCondMock();
-
- @After
- public void teardown() {
- nonEquiMock.clearTableRefCache();
- }
-
- @Test
- public void testSimpleEqual() {
- NonEquiJoinCondition cond1 = nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1");
- Assert.assertEquals(cond1, cond1);
- }
-
- @Test
- public void testNestedCondEqual() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.OR,
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
- nonEquiMock.composite(SqlKind.AND, nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2"),
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2")),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a5", "B.b6")),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"), nonEquiMock.colConstantCompareCond(
- SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR, "SOME TEXT", SqlTypeName.CHAR));
- Assert.assertEquals(cond1, cond1);
- }
-
- @Test
- public void testOrderingIrrelevantCondEqual() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.OR,
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.OR,
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"),
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"));
- Assert.assertEquals(cond1, cond2);
- }
-
- @Test
- public void testComparingWithNull() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.OR,
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"));
- Assert.assertNotEquals(cond1, null);
- }
-
- @Test
- public void testNotEqual() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.OR,
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
- nonEquiMock.composite(SqlKind.AND, nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2"),
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2")),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a5", "B.b6")),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"), nonEquiMock.colConstantCompareCond(
- SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR, "SOME TEXT", SqlTypeName.CHAR));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.OR,
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a2", "B.b1"),
- nonEquiMock.composite(SqlKind.OR, nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b8"),
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a3", "B.b2")),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a5", "B.b6")),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a9", "B.b8"), nonEquiMock.colConstantCompareCond(
- SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR, "SOME TEXT", SqlTypeName.CHAR));
- Assert.assertNotEquals(cond1, cond2);
- }
-
- @Test
- public void testNotEqualOnTable() {
- NonEquiJoinCondition cond1 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "B.b1");
- NonEquiJoinCondition cond2 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "C.b1");
- Assert.assertNotEquals(cond1, cond2);
- }
-
- @Test
- public void testNotEqualOnColumn() {
- NonEquiJoinCondition cond1 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "B.b1");
- NonEquiJoinCondition cond2 = nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "A.a1", "B.b2");
- Assert.assertNotEquals(cond1, cond2);
- }
-
- @Test
- public void testNotEqualOnConstant() {
- NonEquiJoinCondition cond1 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
- "SOME TEXT", SqlTypeName.CHAR);
- NonEquiJoinCondition cond2 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
- "SOME TEXT1", SqlTypeName.CHAR);
- Assert.assertNotEquals(cond1, cond2);
- }
-
- @Test
- public void testNotEqualOnType() {
- NonEquiJoinCondition cond1 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
- "SOME TEXT", SqlTypeName.CHAR);
- NonEquiJoinCondition cond2 = nonEquiMock.colConstantCompareCond(SqlKind.EQUALS, "A.a9", SqlTypeName.CHAR,
- "SOME TEXT", SqlTypeName.INTEGER);
- Assert.assertNotEquals(cond1, cond2);
- }
-
- @Test
- public void testCondNormalizations() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.AND,
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a8", "B.b8"),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a9", "B.b9")),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colOp(SqlKind.IS_NULL, "A.a9")),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN_OR_EQUAL, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a4", "B.b4"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.AND,
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "B.b8", "A.a8"),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "B.b9", "A.a9"),
- nonEquiMock.colOp(SqlKind.IS_NOT_NULL, "A.a9"),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1")),
- nonEquiMock.composite(SqlKind.NOT,
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN_OR_EQUAL, "A.a4", "B.b4")));
- Assert.assertEquals(cond1, cond2);
- }
-
- @Test
- public void testCondNormalizationsOnSqlKindOTHER() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"));
- Assert.assertEquals(cond1, cond2);
- }
-
- @Test
- public void testCondNormalizationsOnSqlKindOTHERNotEqual() {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
- nonEquiMock.colOp("cos", SqlKind.OTHER_FUNCTION, "A.a9"));
- Assert.assertNotEquals(cond1, cond2);
- }
-
- @Test
- public void testNonEquJoinDescComparator() {
-
- Function<NonEquiJoinCondition, JoinDesc> supplierSimpleJoinDesc = (nonEquiJoinCondition) -> {
- JoinDesc joinDesc = new JoinDesc();
- joinDesc.setNonEquiJoinCondition(nonEquiJoinCondition);
- joinDesc.setPrimaryKey(new String[] {});
- joinDesc.setForeignKey(new String[] {});
- joinDesc.setForeignKeyColumns(new TblColRef[] {});
- joinDesc.setPrimaryKeyColumns(new TblColRef[] {});
- joinDesc.setType("LEFT");
- return joinDesc;
- };
- //nonEqual
- {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
- nonEquiMock.colOp("cos", SqlKind.OTHER_FUNCTION, "A.a9"));
- Assert.assertNotEquals(new JoinDescNonEquiCompBean(supplierSimpleJoinDesc.apply(cond1)),
- new JoinDescNonEquiCompBean(supplierSimpleJoinDesc.apply(cond2)));
- }
-
- //equal
- {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"),
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.EQUALS,
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "B.a9"),
- nonEquiMock.colOp("abs", SqlKind.OTHER_FUNCTION, "A.a9"));
- Assert.assertEquals(new JoinDescNonEquiCompBean(supplierSimpleJoinDesc.apply(cond1)),
- new JoinDescNonEquiCompBean(supplierSimpleJoinDesc.apply(cond2)));
- }
- //equal
- {
- NonEquiJoinCondition cond1 = nonEquiMock.composite(SqlKind.AND,
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a8", "B.b8"),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colCompareCond(SqlKind.EQUALS, "A.a9", "B.b9")),
- nonEquiMock.composite(SqlKind.NOT, nonEquiMock.colOp(SqlKind.IS_NULL, "A.a9")),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN_OR_EQUAL, "A.a1", "B.b1"),
- nonEquiMock.colCompareCond(SqlKind.LESS_THAN, "A.a4", "B.b4"));
- NonEquiJoinCondition cond2 = nonEquiMock.composite(SqlKind.AND,
- nonEquiMock.colCompareCond(SqlKind.EQUALS, "B.b8", "A.a8"),
- nonEquiMock.colCompareCond(SqlKind.NOT_EQUALS, "B.b9", "A.a9"),
- nonEquiMock.colOp(SqlKind.IS_NOT_NULL, "A.a9"),
- nonEquiMock.composite(SqlKind.NOT,
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN, "A.a1", "B.b1")),
- nonEquiMock.composite(SqlKind.NOT,
- nonEquiMock.colCompareCond(SqlKind.GREATER_THAN_OR_EQUAL, "A.a4", "B.b4")));
- Assert.assertEquals(new JoinDescNonEquiCompBean(supplierSimpleJoinDesc.apply(cond1)),
- new JoinDescNonEquiCompBean(supplierSimpleJoinDesc.apply(cond2)));
-
- }
- }
-
-}
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index 90cdfa7057..b19afa1616 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -70,7 +70,7 @@ import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.exception.QueryErrorCode;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.AddS3CredentialToSparkBroadcastEventNotifier;
import org.apache.kylin.common.persistence.transaction.TransactionException;
@@ -1406,8 +1406,8 @@ public class TableService extends BasicService {
if (root instanceof IllegalCCExpressionException) {
throw new KylinException(INVALID_COMPUTED_COLUMN_EXPRESSION, String.format(Locale.ROOT,
MsgPicker.getMsg().getReloadTableCcRetry(), root.getMessage(), tableName, columnNames));
- } else if (root instanceof KylinException
- && ((KylinException) root).getErrorCode() == QueryErrorCode.SCD2_COMMON_ERROR.toErrorCode()) {
+ } else if (root instanceof KylinException && ((KylinException) root).getErrorCodeProducer()
+ .getErrorCode() == ErrorCodeServer.SCD2_MODEL_UNKNOWN_EXCEPTION.getErrorCode()) {
throw new KylinException(TABLE_RELOAD_MODEL_RETRY, tableName, columnNames, model.getAlias());
}
throw e;
@@ -1626,8 +1626,8 @@ public class TableService extends BasicService {
context.setRemoveColumns(dataTypeDiff.entriesOnlyOnRight().keySet());
context.setChangedColumns(diff.entriesDiffering().keySet());
context.setChangeTypeColumns(dataTypeDiff.entriesDiffering().keySet());
- context.setTableCommentChanged(!Objects.equals(originTableDesc.getTableComment(), newTableDesc.getTableComment()));
-
+ context.setTableCommentChanged(
+ !Objects.equals(originTableDesc.getTableComment(), newTableDesc.getTableComment()));
if (!context.isNeedProcess()) {
return context;
@@ -1743,8 +1743,7 @@ public class TableService extends BasicService {
val colName = value.getKey();
val colDiff = value.getValue();
- Graphs.reachableNodes(dependencyGraph,
- SchemaNode.ofTableColumn(columnMap.get(colName))).stream()
+ Graphs.reachableNodes(dependencyGraph, SchemaNode.ofTableColumn(columnMap.get(colName))).stream()
.filter(node -> node.getType() == SchemaNodeType.MODEL_MEASURE).forEach(node -> {
String modelAlias = node.getSubject();
String measureId = node.getDetail();
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
index c5462d0c12..094fa8c3c5 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
@@ -47,7 +47,7 @@ public class OlapContextTestUtil {
QueryExec queryExec = new QueryExec(project, KylinConfig.getInstanceFromEnv());
try {
RelNode rel = queryExec.parseAndOptimize(sql);
- QueryContextCutter.selectRealization(rel, reCutBanned);
+ QueryContextCutter.selectRealization(project, rel, reCutBanned);
} catch (NoRealizationFoundException | NoStreamingRealizationFoundException e) {
// When NoRealizationFoundException occurs, do nothing
// because we only need to obtain OlapContexts.
@@ -61,7 +61,7 @@ public class OlapContextTestUtil {
QueryExec queryExec = new QueryExec(project, KylinConfig.getInstanceFromEnv());
try {
RelNode rel = queryExec.parseAndOptimize(sql);
- QueryContextCutter.selectRealization(rel, reCutBanned);
+ QueryContextCutter.selectRealization(project, rel, reCutBanned);
} catch (NoRealizationFoundException e) {
consumer.accept(e);
}
@@ -76,7 +76,7 @@ public class OlapContextTestUtil {
try {
List<RelNode> relNodes = queryExec.postOptimize(rel);
relNodes.forEach(relNode -> {
- QueryContextCutter.selectRealization(relNode, reCutBanned);
+ QueryContextCutter.selectRealization(project, relNode, reCutBanned);
olapContexts.addAll(getOlapContexts());
});
} catch (NoRealizationFoundException e) {
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java
index 40524b1949..dae2b5106c 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java
@@ -43,7 +43,7 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.accept.ContentNegotiationManager;
-import io.kyligence.kap.metadata.user.ManagedUser;
+import org.apache.kylin.metadata.user.ManagedUser;
import lombok.val;
@MetadataInfo(onlyProps = true)
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index 2b1694501c..5fe4a95002 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -47,13 +47,14 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.CommonErrorCode;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.exception.ServerErrorCode;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.ModifyTableNameSqlVisitor;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.util.ThreadUtil;
import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil;
import org.apache.kylin.guava30.shaded.common.base.Throwables;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableBiMap;
@@ -91,14 +92,11 @@ import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.UpdateImpact;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.model.tool.JoinDescNonEquiCompBean;
-import org.apache.kylin.metadata.model.tool.NonEquiJoinConditionVisitor;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
import org.apache.kylin.metadata.model.util.scd2.SCD2CondChecker;
-import org.apache.kylin.metadata.model.util.scd2.SCD2Exception;
-import org.apache.kylin.metadata.model.util.scd2.SCD2NonEquiCondSimplification;
import org.apache.kylin.metadata.model.util.scd2.SCD2SqlConverter;
+import org.apache.kylin.metadata.model.util.scd2.Scd2Simplifier;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinDesc;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -142,7 +140,7 @@ public class ModelSemanticHelper extends BasicService {
nDataModel = JsonUtil.readValue(JsonUtil.writeValueAsIndentString(originModel), NDataModel.class);
nDataModel.setJoinTables(SCD2SimplificationConvertUtil.deepCopyJoinTables(originModel.getJoinTables()));
} catch (IOException e) {
- logger.error("Parse json failed...", e);
+ ThreadUtil.warnKylinStackTrace("Parse json failed...\n");
throw new KylinException(CommonErrorCode.FAILED_PARSE_JSON, e);
}
return nDataModel;
@@ -154,7 +152,7 @@ public class ModelSemanticHelper extends BasicService {
try {
dataModel = JsonUtil.deepCopy(modelRequest, NDataModel.class);
} catch (IOException e) {
- logger.error("Parse json failed...", e);
+ ThreadUtil.warnKylinStackTrace("Parse json failed...\n");
throw new KylinException(CommonErrorCode.FAILED_PARSE_JSON, e);
}
@@ -253,7 +251,7 @@ public class ModelSemanticHelper extends BasicService {
return;
}
- HashSet<JoinDescNonEquiCompBean> scd2NonEquiCondSets = new HashSet<>();
+ Set<JoinDesc> nonEquivJoins = new HashSet<>();
String project = dataModel.getProject();
val projectKylinConfig = NProjectManager.getProjectConfig(project);
@@ -272,7 +270,7 @@ public class ModelSemanticHelper extends BasicService {
//1. check scd2 turn on when non-equi join exists
if (!isScd2Enabled) {
- throw new KylinException(QueryErrorCode.SCD2_SAVE_MODEL_WHEN_DISABLED, "please turn on scd2 config");
+ throw new KylinException(ErrorCodeServer.PROJECT_SCD2_IS_NOT_ALLOWED);
}
//2. check request equi join condition
@@ -283,7 +281,7 @@ public class ModelSemanticHelper extends BasicService {
requestJoinDesc.getSimplifiedNonEquiJoinConditions());
final JoinDesc analyzedJoin = deriveJoins(queryExec, scd2Sql);
// restore table alias in non-equi conditions
- final NonEquiJoinCondition nonEquiCondWithAliasRestored = new NonEquiJoinConditionVisitor() {
+ final NonEquiJoinCondition nonEquiCondWithAliasRestored = new NonEquiJoinCondition.NeqConditionVisitor() {
@Override
public NonEquiJoinCondition visitColumn(NonEquiJoinCondition cond) {
TableRef originalTableRef;
@@ -310,40 +308,44 @@ public class ModelSemanticHelper extends BasicService {
//4. update dataModel
try {
-
- SCD2NonEquiCondSimplification.INSTANCE.convertToSimplifiedSCD2Cond(analyzedJoin);
+ Scd2Simplifier.INSTANCE.simplifyScd2Conditions(analyzedJoin);
joinWithoutNonEquivInfo.setNonEquiJoinCondition(analyzedJoin.getNonEquiJoinCondition());
joinWithoutNonEquivInfo.setForeignTable(analyzedJoin.getForeignTable());
joinWithoutNonEquivInfo.setPrimaryTable(analyzedJoin.getPrimaryTable());
- } catch (SCD2Exception e) {
- logger.error("Update datamodel failed...", e);
- throw new KylinException(QueryErrorCode.SCD2_COMMON_ERROR, Throwables.getRootCause(e).getMessage());
+ } catch (Exception e) {
+ ThreadUtil.warnKylinStackTrace("Update model failed...\n");
+ if (e instanceof KylinException) {
+ throw e;
+ }
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_UNKNOWN_EXCEPTION,
+ Throwables.getRootCause(e).getMessage());
}
//5. check same join conditions
- if (scd2NonEquiCondSets.contains(new JoinDescNonEquiCompBean(requestJoinDesc))) {
- throw new KylinException(QueryErrorCode.SCD2_DUPLICATE_CONDITION, "duplicate join edge");
+ if (nonEquivJoins.contains(requestJoinDesc)) {
+ throw new KylinException(ErrorCodeServer.DUPLICATE_MODEL_JOIN_CONDITIONS);
} else {
- scd2NonEquiCondSets.add(new JoinDescNonEquiCompBean(requestJoinDesc));
+ nonEquivJoins.add(requestJoinDesc);
}
-
}
-
}
private JoinDesc deriveJoins(QueryExec queryExec, String sql) {
List<OLAPContext> contexts = queryExec.deriveOlapContexts(sql);
- Optional<RuntimeException> th;
- if (contexts.size() == 0) {
- th = Optional.of(new SCD2Exception("Failed to extract joins from the input sql: " + sql));
+ Optional<KylinException> th;
+ if (contexts.isEmpty()) {
+ th = Optional.of(new KylinException(ErrorCodeServer.SCD2_MODEL_UNKNOWN_EXCEPTION,
+ "Failed to extract joins from the input sql: " + sql));
} else if (contexts.size() > 1) {
- th = Optional.of(new SCD2Exception("Non-equiv-join conditions were split. the input sql is: " + sql));
+ th = Optional.of(new KylinException(ErrorCodeServer.SCD2_MODEL_UNKNOWN_EXCEPTION,
+ "Small sub-queries were split from the input sql: " + sql));
} else {
OLAPContext ctx = contexts.get(0);
if (ctx.joins.size() == 1) {
return ctx.joins.get(0);
}
- th = Optional.of(new SCD2Exception("Non-equiv-join conditions were split. the input sql is: " + sql));
+ th = Optional.of(new KylinException(ErrorCodeServer.SCD2_MODEL_UNKNOWN_EXCEPTION,
+ "Non-equiv-join conditions were split. the input sql is: " + sql));
}
throw th.get();
}
@@ -352,14 +354,14 @@ public class ModelSemanticHelper extends BasicService {
if (!SCD2CondChecker.INSTANCE.checkSCD2EquiJoinCond(requestJoinDesc.getForeignKey(),
requestJoinDesc.getPrimaryKey())) {
- throw new KylinException(QueryErrorCode.SCD2_EMPTY_EQUI_JOIN, "SCD2 must have one equi join conditon");
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_REQUIRES_AT_LEAST_ONE_EQUAL_CONDITION);
}
if (!SCD2CondChecker.INSTANCE
.checkSCD2NonEquiJoinCondPair(requestJoinDesc.getSimplifiedNonEquiJoinConditions())) {
- throw new KylinException(QueryErrorCode.SCD2_DUPLICATE_FK_PK_PAIR, "SCD2 non-equi condition must be pair");
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_REQUIRES_AT_LEAST_ONE_NON_EQUAL_CONDITION);
}
if (!SCD2CondChecker.INSTANCE.checkFkPkPairUnique(requestJoinDesc)) {
- throw new KylinException(QueryErrorCode.SCD2_DUPLICATE_FK_PK_PAIR, "SCD2 condition must be unqiue");
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_PK_FK_UNIQUE_CHECK_FAILED);
}
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/util/SCD2SimplificationConvertUtil.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/util/SCD2SimplificationConvertUtil.java
index f9b4b6d559..b9bd60edb9 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/util/SCD2SimplificationConvertUtil.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/util/SCD2SimplificationConvertUtil.java
@@ -18,24 +18,25 @@
package org.apache.kylin.rest.util;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import javax.annotation.Nonnull;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.exception.QueryErrorCode;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.util.scd2.SCD2Exception;
-import org.apache.kylin.metadata.model.util.scd2.SCD2NonEquiCondSimplification;
+import org.apache.kylin.metadata.model.util.scd2.Scd2Simplifier;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinDesc;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
public class SCD2SimplificationConvertUtil {
+ private SCD2SimplificationConvertUtil() {
+ }
/**
* fill simplified cond according to joinTables
@@ -57,20 +58,17 @@ public class SCD2SimplificationConvertUtil {
}
try {
- SimplifiedJoinDesc convertedJoinDesc = SCD2NonEquiCondSimplification.INSTANCE
- .convertToSimplifiedSCD2Cond(joinDesc);
-
+ SimplifiedJoinDesc convertedJoinDesc = Scd2Simplifier.INSTANCE.simplifyScd2Conditions(joinDesc);
SimplifiedJoinDesc responseJoinDesc = simplifiedJoinTableDescs.get(i).getSimplifiedJoinDesc();
responseJoinDesc
.setSimplifiedNonEquiJoinConditions(convertedJoinDesc.getSimplifiedNonEquiJoinConditions());
responseJoinDesc.setForeignKey(convertedJoinDesc.getForeignKey());
responseJoinDesc.setPrimaryKey(convertedJoinDesc.getPrimaryKey());
- } catch (SCD2Exception e) {
- throw new KylinException(QueryErrorCode.SCD2_COMMON_ERROR, "only support scd2 join condition");
+ } catch (KylinException e) {
+ throw new KylinException(ErrorCodeServer.SCD2_MODEL_UNKNOWN_EXCEPTION,
+ "only support scd2 join condition");
}
-
}
-
}
/**
@@ -81,7 +79,7 @@ public class SCD2SimplificationConvertUtil {
*/
public static List<SimplifiedJoinTableDesc> simplifiedJoinTablesConvert(List<JoinTableDesc> joinTables) {
if (Objects.isNull(joinTables)) {
- return null;
+ return Collections.emptyList();
}
List<SimplifiedJoinTableDesc> simplifiedJoinTableDescs;
@@ -89,7 +87,7 @@ public class SCD2SimplificationConvertUtil {
simplifiedJoinTableDescs = Arrays.asList(
JsonUtil.readValue(JsonUtil.writeValueAsString(joinTables), SimplifiedJoinTableDesc[].class));
} catch (Exception e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
fillSimplifiedCond(simplifiedJoinTableDescs, joinTables);
@@ -106,14 +104,14 @@ public class SCD2SimplificationConvertUtil {
public static List<JoinTableDesc> convertSimplified2JoinTables(
List<SimplifiedJoinTableDesc> simplifiedJoinTableDescs) {
if (Objects.isNull(simplifiedJoinTableDescs)) {
- return null;
+ return Collections.emptyList();
}
List<JoinTableDesc> joinTableDescs;
try {
joinTableDescs = Arrays.asList(
JsonUtil.readValue(JsonUtil.writeValueAsString(simplifiedJoinTableDescs), JoinTableDesc[].class));
} catch (Exception e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
return joinTableDescs;
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/request/ModelRequestTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/request/ModelRequestTest.java
index e4647b9c80..2b75393503 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/request/ModelRequestTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/request/ModelRequestTest.java
@@ -107,7 +107,7 @@ class ModelRequestTest {
List<String> actualNonEqualForeignKeys = modelRequest.getSimplifiedJoinTableDescs().stream()
.map(SimplifiedJoinTableDesc::getSimplifiedJoinDesc)
.map(SimplifiedJoinDesc::getSimplifiedNonEquiJoinConditions).filter(Objects::nonNull)
- .flatMap(Collection::stream).map(NonEquiJoinCondition.SimplifiedNonEquiJoinCondition::getForeignKey)
+ .flatMap(Collection::stream).map(NonEquiJoinCondition.SimplifiedJoinCondition::getForeignKey)
.collect(Collectors.toList());
List<String> expectedNonEqualForeignKey = Collections.singletonList("TEST_TABLE3.COLUMN2");
Assertions.assertEquals(expectedNonEqualForeignKey, actualNonEqualForeignKeys);
@@ -115,7 +115,7 @@ class ModelRequestTest {
List<String> actualNonEqualPrimaryKeys = modelRequest.getSimplifiedJoinTableDescs().stream()
.map(SimplifiedJoinTableDesc::getSimplifiedJoinDesc)
.map(SimplifiedJoinDesc::getSimplifiedNonEquiJoinConditions).filter(Objects::nonNull)
- .flatMap(Collection::stream).map(NonEquiJoinCondition.SimplifiedNonEquiJoinCondition::getPrimaryKey)
+ .flatMap(Collection::stream).map(NonEquiJoinCondition.SimplifiedJoinCondition::getPrimaryKey)
.collect(Collectors.toList());
List<String> expectedNonEqualPrimaryKey = Collections.singletonList("TEST_TABLE3.COLUMN3");
Assertions.assertEquals(expectedNonEqualPrimaryKey, actualNonEqualPrimaryKeys);
@@ -191,7 +191,7 @@ class ModelRequestTest {
simplifiedJoinDesc.setPrimaryKey(new String[] { "TEST_TABLE3.cOLUMn3" });
simplifiedList.add(simplified);
- val condition = new NonEquiJoinCondition.SimplifiedNonEquiJoinCondition();
+ val condition = new NonEquiJoinCondition.SimplifiedJoinCondition();
condition.setPrimaryKey("TEST_TABLE3.cOLUMn3");
condition.setForeignKey("TEST_TABLE3.COLUMN2");
simplifiedJoinDesc.setSimplifiedNonEquiJoinConditions(Collections.singletonList(condition));
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
index 0c236edd40..692629dafe 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/BaseIndexTest.java
@@ -31,6 +31,8 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexEntity.Source;
@@ -59,10 +61,6 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-
public class BaseIndexTest extends SourceTestCase {
private static final String COMMON_MODEL_ID = "b780e4e4-69af-449e-b09f-05c90dfa04b6";
@@ -94,6 +92,7 @@ public class BaseIndexTest extends SourceTestCase {
ReflectionTestUtils.setField(indexPlanService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
modelService.setSemanticUpdater(semanticService);
modelService.setIndexPlanService(indexPlanService);
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java
index 103ac90988..da66e04aa1 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java
@@ -109,6 +109,7 @@ public class ModelServiceBrokenRepairTest extends SourceTestCase {
ReflectionTestUtils.setField(indexPlanService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
modelService.setSemanticUpdater(semanticService);
modelService.setModelQuerySupporter(modelQueryService);
modelService.setIndexPlanService(indexPlanService);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java
index 07fa1fe8bc..429f5aa882 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceSemanticUpdateTest.java
@@ -17,9 +17,6 @@
*/
package org.apache.kylin.rest.service;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SIMPLIFIED_MEASURES_MISSING_ID;
-import static org.hamcrest.Matchers.is;
-
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -37,6 +34,7 @@ import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
@@ -86,7 +84,6 @@ import org.apache.kylin.rest.response.SimplifiedMeasure;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclUtil;
import org.apache.kylin.rest.util.SCD2SimplificationConvertUtil;
-import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -107,7 +104,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
- private static final String MODEL_ID = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ private static final String BASIC_MODEL = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ private static final String INNER_MODEL = "741ca86a-1f13-46da-a59f-95fb68615e3a";
@InjectMocks
private ModelService modelService = Mockito.spy(new ModelService());
@@ -127,8 +125,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Mock
protected IUserGroupService userGroupService = Mockito.spy(NUserGroupService.class);
- @Before
- public void setupResource() throws Exception {
+ private void setupResource() {
overwriteSystemProp("HADOOP_USER_NAME", "root");
createTestMetadata();
modelService.setSemanticUpdater(semanticService);
@@ -136,14 +133,8 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
modelService.setIndexPlanService(indexPlanService);
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
- modelMgr.updateDataModel(MODEL_ID, model -> {
- model.setManagementType(ManagementType.MODEL_BASED);
- });
- modelMgr.updateDataModel("741ca86a-1f13-46da-a59f-95fb68615e3a", model -> {
- model.setManagementType(ManagementType.MODEL_BASED);
- });
-
- ReflectionTestUtils.setField(indexPlanService, "aclEvaluate", aclEvaluate);
+ modelMgr.updateDataModel(BASIC_MODEL, model -> model.setManagementType(ManagementType.MODEL_BASED));
+ modelMgr.updateDataModel(INNER_MODEL, model -> model.setManagementType(ManagementType.MODEL_BASED));
}
private String getProject() {
@@ -152,12 +143,15 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Before
public void setup() {
+ setupResource();
SparkJobFactoryUtils.initJobFactory();
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
+ ReflectionTestUtils.setField(indexPlanService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(aclEvaluate, "aclUtil", Mockito.spy(AclUtil.class));
ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
try {
new JdbcRawRecStore(getTestConfig());
} catch (Exception e) {
@@ -231,11 +225,11 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
ModelRequest requestToVerify = newSemanticRequest();
Assert.assertEquals(colIdOfCC, requestToVerify.getColumnIdByColumnName(ccColName));
- NamedColumn dimensionToVerify = requestToVerify.getSimplifiedDimensions().stream()
- .filter(col -> col.getId() == colIdOfCC).findFirst().get();
- Assert.assertNotNull(dimensionToVerify);
- Assert.assertEquals("TEST_DIM_WITH_CC", dimensionToVerify.getName());
- Assert.assertEquals(ColumnStatus.DIMENSION, dimensionToVerify.getStatus());
+ Optional<NamedColumn> dimensionToVerify = requestToVerify.getSimplifiedDimensions().stream()
+ .filter(col -> col.getId() == colIdOfCC).findFirst();
+ Assert.assertTrue(dimensionToVerify.isPresent());
+ Assert.assertEquals("TEST_DIM_WITH_CC", dimensionToVerify.get().getName());
+ Assert.assertEquals(ColumnStatus.DIMENSION, dimensionToVerify.get().getStatus());
}
// Add measure which uses TEST_CC_1
@@ -260,35 +254,35 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
modelService.updateDataModelSemantic(request.getProject(), request);
NDataModel model = getTestModel();
- Measure measure = model.getAllMeasures().stream().filter(m -> m.getName().equals("TEST_MEASURE_WITH_CC"))
- .findFirst().get();
- Assert.assertNotNull(measure);
- measureIdOfCC = measure.getId();
- Assert.assertTrue(measure.getFunction().isSum());
- Assert.assertEquals(ccColName, measure.getFunction().getParameters().get(0).getValue());
+ Optional<Measure> measure = model.getAllMeasures().stream()
+ .filter(m -> m.getName().equals("TEST_MEASURE_WITH_CC")).findFirst();
+ Assert.assertTrue(measure.isPresent());
+ measureIdOfCC = measure.get().getId();
+ Assert.assertTrue(measure.get().getFunction().isSum());
+ Assert.assertEquals(ccColName, measure.get().getFunction().getParameters().get(0).getValue());
}
// Update TEST_CC_1's definition, named column and measure will be updated
{
ModelRequest request = newSemanticRequest();
- ComputedColumnDesc ccDesc = request.getComputedColumnDescs().stream()
- .filter(cc -> cc.getColumnName().equals("TEST_CC_1")).findFirst().get();
- Assert.assertNotNull(ccDesc);
- ccDesc.setExpression("1 + 2");
+ Optional<ComputedColumnDesc> ccDesc = request.getComputedColumnDescs().stream()
+ .filter(cc -> cc.getColumnName().equals("TEST_CC_1")).findFirst();
+ Assert.assertTrue(ccDesc.isPresent());
+ ccDesc.get().setExpression("1 + 2");
modelService.updateDataModelSemantic(request.getProject(), request);
NDataModel model = getTestModel();
- NamedColumn originalColumn = model.getAllNamedColumns().stream().filter(col -> col.getId() == colIdOfCC)
- .findFirst().get();
- Assert.assertNotNull(originalColumn);
- Assert.assertEquals("TEST_DIM_WITH_CC", originalColumn.getName());
- Assert.assertEquals(ColumnStatus.DIMENSION, originalColumn.getStatus());
-
- Measure originalMeasure = model.getAllMeasures().stream().filter(m -> m.getId() == measureIdOfCC)
- .findFirst().get();
- Assert.assertNotNull(originalMeasure);
- Assert.assertEquals("TEST_MEASURE_WITH_CC", originalMeasure.getName());
- Assert.assertFalse(originalMeasure.isTomb());
+ Optional<NamedColumn> originalColumn = model.getAllNamedColumns().stream()
+ .filter(col -> col.getId() == colIdOfCC).findFirst();
+ Assert.assertTrue(originalColumn.isPresent());
+ Assert.assertEquals("TEST_DIM_WITH_CC", originalColumn.get().getName());
+ Assert.assertEquals(ColumnStatus.DIMENSION, originalColumn.get().getStatus());
+
+ Optional<Measure> originalMeasure = model.getAllMeasures().stream().filter(m -> m.getId() == measureIdOfCC)
+ .findFirst();
+ Assert.assertTrue(originalMeasure.isPresent());
+ Assert.assertEquals("TEST_MEASURE_WITH_CC", originalMeasure.get().getName());
+ Assert.assertFalse(originalMeasure.get().isTomb());
}
// Remove TEST_CC_1, all related should be moved to tomb
@@ -303,10 +297,14 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
modelService.updateDataModelSemantic(request.getProject(), request);
NDataModel model = getTestModel();
- Assert.assertFalse(model.getAllNamedColumns().stream().filter(c -> c.getId() == colIdOfCC).findFirst().get()
- .isExist());
- Assert.assertTrue(
- model.getAllMeasures().stream().filter(m -> m.getId() == measureIdOfCC).findFirst().get().isTomb());
+ Optional<NamedColumn> first = model.getAllNamedColumns().stream().filter(c -> c.getId() == colIdOfCC)
+ .findFirst();
+ Assert.assertTrue(first.isPresent());
+ Assert.assertFalse(first.get().isExist());
+ Optional<Measure> second = model.getAllMeasures().stream().filter(m -> m.getId() == measureIdOfCC)
+ .findFirst();
+ Assert.assertTrue(second.isPresent());
+ Assert.assertTrue(second.get().isTomb());
}
}
@@ -327,9 +325,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
.getIndexPlan(getTestModel().getUuid());
UnitOfWork.doInTransactionWithRetry(() -> {
NIndexPlanManager.getInstance(getTestConfig(), getProject()).updateIndexPlan(indexPlan.getUuid(),
- copyForWrite -> {
- copyForWrite.setIndexes(new ArrayList<>());
- });
+ copyForWrite -> copyForWrite.setIndexes(new ArrayList<>()));
return 0;
}, getProject());
modelService.updateDataModelSemantic(getProject(), request);
@@ -377,7 +373,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
}, getProject());
modelService.updateDataModelSemantic(getProject(), request);
val modelMgr = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
- val model = modelMgr.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ val model = modelMgr.getDataModelDesc(BASIC_MODEL);
Assert.assertNull(model.getEffectiveMeasures().get(100010));
Assert.assertEquals(1, model.getAllMeasures().stream()
.filter(m -> m.getFunction().getReturnType().equals("hllc(12)")).count());
@@ -397,24 +393,18 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testRenameTableAlias() throws Exception {
var request = newSemanticRequest();
- val OLD_ALIAS = "TEST_ORDER";
- val NEW_ALIAS = "NEW_ALIAS";
- val colCount = request.getAllNamedColumns().stream().filter(n -> n.getAliasDotColumn().startsWith("TEST_ORDER"))
- .count();
- request = changeAlias(request, OLD_ALIAS, NEW_ALIAS);
+ request = changeAlias(request, "TEST_ORDER", "NEW_ALIAS");
modelService.updateDataModelSemantic(getProject(), request);
val model = getTestModel();
val tombCount = model.getAllNamedColumns().stream().filter(n -> n.getAliasDotColumn().startsWith("TEST_ORDER"))
- .peek(col -> {
- Assert.assertEquals(ColumnStatus.TOMB, col.getStatus());
- }).count();
+ .peek(col -> Assert.assertEquals(ColumnStatus.TOMB, col.getStatus())).count();
Assert.assertEquals(0, tombCount);
val otherTombCount = model.getAllNamedColumns().stream()
.filter(n -> !n.getAliasDotColumn().startsWith("TEST_ORDER")).filter(nc -> !nc.isExist()).count();
Assert.assertEquals(1, otherTombCount);
Assert.assertEquals(202, model.getAllNamedColumns().size());
- val executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ val executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(0, executables.size());
}
@@ -471,7 +461,8 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
modelService.updateDataModelSemantic(project, updateRequest);
Assert.fail();
} catch (KylinException e) {
- Assert.assertEquals(SIMPLIFIED_MEASURES_MISSING_ID.getErrorCode().getCode(), e.getErrorCodeString());
+ Assert.assertEquals(ErrorCodeServer.SIMPLIFIED_MEASURES_MISSING_ID.getErrorCode().getCode(),
+ e.getErrorCodeString());
}
}
@@ -632,14 +623,13 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
// reserve cc & corresponding column
String ccDealYear = "DEAL_YEAR";
- ComputedColumnDesc ccDesc = request.getComputedColumnDescs().stream()
- .filter(cc -> ccDealYear.equals(cc.getColumnName())).findFirst().orElse(null);
- NamedColumn ccCol = request.getAllNamedColumns().stream().filter(c -> {
- assert ccDesc != null;
- return c.getAliasDotColumn().equals(ccDesc.getFullName());
- }).findFirst().orElse(null);
- Assert.assertNotNull(ccDesc);
- Assert.assertNotNull(ccCol);
+ Optional<ComputedColumnDesc> ccDescOptional = request.getComputedColumnDescs().stream()
+ .filter(cc -> ccDealYear.equals(cc.getColumnName())).findFirst();
+ Assert.assertTrue(ccDescOptional.isPresent());
+ ComputedColumnDesc ccDesc = ccDescOptional.get();
+ Optional<NamedColumn> ccCol = request.getAllNamedColumns().stream()
+ .filter(c -> c.getAliasDotColumn().equals(ccDesc.getFullName())).findFirst();
+ Assert.assertTrue(ccCol.isPresent());
// set "TEST_KYLIN_FACT.PRICE" as dimension and rename
String colPrice = "TEST_KYLIN_FACT.PRICE";
@@ -676,8 +666,8 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
Assert.assertEquals("PRICE2", model.getNameByColumnId(preservedId));
Assert.assertNull(model.getEffectiveDimensions().get(25));
Assert.assertFalse(model.getComputedColumnNames().contains(ccDealYear));
- Assert.assertNull(model.getEffectiveDimensions().get(ccCol.getId()));
- Assert.assertNull(model.getEffectiveCols().get(ccCol.getId()));
+ Assert.assertNull(model.getEffectiveDimensions().get(ccCol.get().getId()));
+ Assert.assertNull(model.getEffectiveCols().get(ccCol.get().getId()));
// rename & update again
request.getAllNamedColumns().stream() //
@@ -695,28 +685,28 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
.filter(c -> c.getAliasDotColumn().equals(ccDesc.getFullName())).filter(NamedColumn::isExist)
.findFirst().orElse(null);
Assert.assertNotNull(newCcCol);
- Assert.assertNotEquals(ccCol.getId(), newCcCol.getId());
+ Assert.assertNotEquals(ccCol.get().getId(), newCcCol.getId());
}
@Test
public void testModelAddDimensions() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
String project = getProject();
val modelMgr = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- val model = modelMgr.getDataModelDesc(modelId);
+ val model = modelMgr.getDataModelDesc(BASIC_MODEL);
// delete all indexes and agg groups
val indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- val LayoutList = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream().map(LayoutEntity::getId)
+ val LayoutList = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayouts().stream().map(LayoutEntity::getId)
.collect(Collectors.toSet());
- indexPlanService.removeIndexes(getProject(), modelId, LayoutList);
- indexPlanService.updateRuleBasedCuboid(getProject(), UpdateRuleBasedCuboidRequest.builder()
- .project(getProject()).modelId(modelId).aggregationGroups(java.util.Collections.emptyList()).build());
+ indexPlanService.removeIndexes(getProject(), BASIC_MODEL, LayoutList);
+ indexPlanService.updateRuleBasedCuboid(getProject(),
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(java.util.Collections.emptyList()).build());
// modify dimensions and measures, 2 dimensions and 1 measures
val request = JsonUtil.readValue(JsonUtil.writeValueAsString(model), ModelRequest.class);
request.setProject(project);
- request.setUuid(modelId);
+ request.setUuid(BASIC_MODEL);
request.setSimplifiedDimensions(model.getAllNamedColumns().stream().filter(NDataModel.NamedColumn::isDimension)
.filter(f -> f.getAliasDotColumn().contains("TEST_KYLIN_FACT")).collect(Collectors.toList())
.subList(0, 2));
@@ -739,17 +729,16 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
newAggregationGroup.setSelectRule(selectRule);
indexPlanService.updateRuleBasedCuboid(getProject(),
- UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(modelId)
- .aggregationGroups(Lists.<NAggregationGroup> newArrayList(newAggregationGroup)).build());
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(Lists.newArrayList(newAggregationGroup)).build());
- val originPlan1 = indexPlanManager.getIndexPlan(modelId);
- Long bAL = indexPlanManager.getIndexPlan(modelId).getBaseAggLayout().getId();
- java.util.Set<Long> allLIs = indexPlanManager.getIndexPlan(modelId).getAllLayoutIds(false);
+ Long bAL = indexPlanManager.getIndexPlan(BASIC_MODEL).getBaseAggLayout().getId();
+ java.util.Set<Long> allLIs = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayoutIds(false);
- // add new dimenssion
+ // add new dimension
val request3 = JsonUtil.readValue(JsonUtil.writeValueAsString(model), ModelRequest.class);
request3.setProject(project);
- request3.setUuid(modelId);
+ request3.setUuid(BASIC_MODEL);
request3.setSimplifiedDimensions(model.getAllNamedColumns().stream().filter(NDataModel.NamedColumn::isDimension)
.filter(f -> f.getAliasDotColumn().contains("TEST_KYLIN_FACT")).collect(Collectors.toList())
.subList(0, 3));
@@ -761,9 +750,9 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val requestj3 = JsonUtil.readValue(JsonUtil.writeValueAsString(request3), ModelRequest.class);
modelService.updateDataModelSemantic(getProject(), requestj3);
- // get all layout, baselaout
- Long bAL2 = indexPlanManager.getIndexPlan(modelId).getBaseAggLayout().getId();
- java.util.Set<Long> allLIs2 = indexPlanManager.getIndexPlan(modelId).getAllLayoutIds(false);
+ // get all layout, baseLayout
+ Long bAL2 = indexPlanManager.getIndexPlan(BASIC_MODEL).getBaseAggLayout().getId();
+ java.util.Set<Long> allLIs2 = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayoutIds(false);
Assert.assertNotEquals(bAL, bAL2);
Assert.assertNotEquals(allLIs.size(), allLIs2.size());
@@ -771,17 +760,18 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
// for test coverage
// delete base agg layout
- val LayoutListBase = indexPlanManager.getIndexPlan(modelId).getBaseAggLayout().getId();
+ val LayoutListBase = indexPlanManager.getIndexPlan(BASIC_MODEL).getBaseAggLayout().getId();
HashSet<Long> toDeleteIds = new HashSet<>();
toDeleteIds.add(LayoutListBase);
- indexPlanService.removeIndexes(getProject(), modelId, toDeleteIds);
- indexPlanService.updateRuleBasedCuboid(getProject(), UpdateRuleBasedCuboidRequest.builder()
- .project(getProject()).modelId(modelId).aggregationGroups(java.util.Collections.emptyList()).build());
+ indexPlanService.removeIndexes(getProject(), BASIC_MODEL, toDeleteIds);
+ indexPlanService.updateRuleBasedCuboid(getProject(),
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(java.util.Collections.emptyList()).build());
// add new dimenssion
val request4 = JsonUtil.readValue(JsonUtil.writeValueAsString(model), ModelRequest.class);
request4.setProject(project);
- request4.setUuid(modelId);
+ request4.setUuid(BASIC_MODEL);
request4.setSimplifiedDimensions(model.getAllNamedColumns().stream().filter(NDataModel.NamedColumn::isDimension)
.filter(f -> f.getAliasDotColumn().contains("TEST_KYLIN_FACT")).collect(Collectors.toList())
.subList(0, 4));
@@ -840,12 +830,10 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testRemoveDimensionOfDirtyModel() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
-
- UpdateRuleBasedCuboidRequest.convertToRequest(getProject(), modelId, false, new RuleBasedIndex());
+ UpdateRuleBasedCuboidRequest.convertToRequest(getProject(), BASIC_MODEL, false, new RuleBasedIndex());
NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- indexPlanManager.updateIndexPlan(modelId, copyForWrite -> {
- IndexPlan indexPlan = indexPlanManager.getIndexPlan(modelId);
+ indexPlanManager.updateIndexPlan(BASIC_MODEL, copyForWrite -> {
+ IndexPlan indexPlan = indexPlanManager.getIndexPlan(BASIC_MODEL);
RuleBasedIndex ruleBasedIndex = new RuleBasedIndex();
ruleBasedIndex.getMeasures().addAll(Lists.newArrayList(100000, 101000));
ruleBasedIndex.setSchedulerVersion(2);
@@ -858,10 +846,9 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
thrown.expectMessage(
"The dimension TEST_KYLIN_FACT.LSTG_FORMAT_NAME is referenced by indexes or aggregate groups. "
+ "Please go to the Data Asset - Model - Index page to view, delete referenced aggregate groups and indexes.");
- val request = newSemanticRequest(modelId);
+ val request = newSemanticRequest(BASIC_MODEL);
request.getSimplifiedDimensions().removeIf(col -> col.getName().equalsIgnoreCase("LSTG_FORMAT_NAME"));
modelService.updateDataModelSemantic(getProject(), request);
-
}
@Test
@@ -877,7 +864,6 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testRemoveCCInShardCol() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
// ensure model has an agg group
NAggregationGroup newAggregationGroup = new NAggregationGroup();
newAggregationGroup.setIncludes(new Integer[] { 0 });
@@ -888,21 +874,21 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
selectRule.jointDims = new Integer[0][0];
newAggregationGroup.setSelectRule(selectRule);
indexPlanService.updateRuleBasedCuboid(getProject(),
- UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(modelId)
- .aggregationGroups(Lists.<NAggregationGroup> newArrayList(newAggregationGroup)).build());
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(Lists.newArrayList(newAggregationGroup)).build());
val indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- val indexList = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream().map(LayoutEntity::getId)
+ val indexList = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayouts().stream().map(LayoutEntity::getId)
.collect(Collectors.toSet());
- indexPlanService.removeIndexes(getProject(), modelId, indexList);
+ indexPlanService.removeIndexes(getProject(), BASIC_MODEL, indexList);
val shardReq = new AggShardByColumnsRequest();
- shardReq.setModelId(modelId);
+ shardReq.setModelId(BASIC_MODEL);
shardReq.setProject(getProject());
shardReq.setShardByColumns(Lists.newArrayList("TEST_KYLIN_FACT.NEST5"));
indexPlanService.updateShardByColumns(getProject(), shardReq);
- var request = newSemanticRequest(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
request.getComputedColumnDescs().removeIf(c -> ("NEST5").equals(c.getColumnName()));
request.getSimplifiedDimensions()
.removeIf(column -> column.getAliasDotColumn().equalsIgnoreCase("TEST_KYLIN_FACT.NEST5"));
@@ -910,13 +896,11 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
.filter(column -> column.getAliasDotColumn().equalsIgnoreCase("TEST_KYLIN_FACT.NEST5"))
.forEach(column -> column.setStatus(ColumnStatus.TOMB));
modelService.updateDataModelSemantic(getProject(), request);
-
- Assert.assertEquals(0, indexPlanService.getShardByColumns(getProject(), modelId).getShardByColumns().size());
+ Assert.assertTrue(indexPlanService.getShardByColumns(getProject(), BASIC_MODEL).getShardByColumns().isEmpty());
}
@Test
public void testRemoveCCExistInTableIndexWithAggGroup() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
// ensure model has an agg group
NAggregationGroup newAggregationGroup = new NAggregationGroup();
newAggregationGroup.setIncludes(new Integer[] { 0 });
@@ -927,11 +911,11 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
selectRule.jointDims = new Integer[0][0];
newAggregationGroup.setSelectRule(selectRule);
indexPlanService.updateRuleBasedCuboid(getProject(),
- UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(modelId)
- .aggregationGroups(Lists.<NAggregationGroup> newArrayList(newAggregationGroup)).build());
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(Lists.newArrayList(newAggregationGroup)).build());
// remove cc TEST_KYLIN_FACT.NEST5
- var request = newSemanticRequest(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
request.getComputedColumnDescs().removeIf(c -> ("NEST5").equals(c.getColumnName()));
request.getSimplifiedDimensions()
.removeIf(column -> column.getAliasDotColumn().equalsIgnoreCase("TEST_KYLIN_FACT.NEST5"));
@@ -946,10 +930,9 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testModifyCCExistInTableIndex() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
val indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
- var request = newSemanticRequest(modelId);
- val originPlan = indexPlanManager.getIndexPlan(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
+ val originPlan = indexPlanManager.getIndexPlan(BASIC_MODEL);
val nest5 = request.getColumnIdByColumnName("TEST_KYLIN_FACT.NEST5");
val transId = request.getColumnIdByColumnName("TEST_KYLIN_FACT.TRANS_ID");
val siteName = request.getColumnIdByColumnName("TEST_SITES.SITE_NAME");
@@ -959,11 +942,12 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
.findFirst().map(LayoutEntity::getId).orElse(-1L);
// modify expression of cc TEST_KYLIN_FACT.NEST5
val originCC = request.getComputedColumnDescs().stream().filter(c -> ("NEST5").equals(c.getColumnName()))
- .findFirst().orElse(null);
- originCC.setExpression(originCC.getExpression() + "+1");
+ .findFirst();
+ Assert.assertTrue(originCC.isPresent());
+ originCC.get().setExpression(originCC.get().getExpression() + "+1");
modelService.updateDataModelSemantic(getProject(), request);
// new indexes
- val newPlan = indexPlanManager.getIndexPlan(modelId);
+ val newPlan = indexPlanManager.getIndexPlan(BASIC_MODEL);
val newIndexId = newPlan.getAllLayouts().stream().filter(l -> l.getColOrder().containsAll(indexCol)).findFirst()
.map(LayoutEntity::getId).orElse(-2L);
Assert.assertTrue(newIndexId > oldIndexId);
@@ -971,10 +955,9 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testModifyCCExistInAggIndex() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
val indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
// create measure
- var request = newSemanticRequest(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
val newMeasure1 = new SimplifiedMeasure();
newMeasure1.setName("NEST5_SUM");
newMeasure1.setExpression("SUM");
@@ -986,7 +969,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
newMeasure1.setReturnType("decimal(38, 0)");
modelService.updateDataModelSemantic(getProject(), request);
// get dim and measure id
- request = newSemanticRequest(modelId);
+ request = newSemanticRequest(BASIC_MODEL);
val transId = request.getColumnIdByColumnName("TEST_KYLIN_FACT.TRANS_ID");
val nest5SumId = request.getSimplifiedMeasures().stream().filter(m -> "NEST5_SUM".equals(m.getName()))
.mapToInt(SimplifiedMeasure::getId).findFirst().orElse(-1);
@@ -1000,29 +983,29 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
selectRule.jointDims = new Integer[0][0];
newAggregationGroup.setSelectRule(selectRule);
indexPlanService.updateRuleBasedCuboid(getProject(),
- UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(modelId)
- .aggregationGroups(Lists.<NAggregationGroup> newArrayList(newAggregationGroup)).build());
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(Lists.newArrayList(newAggregationGroup)).build());
// old indexes
val indexCol = Arrays.asList(transId, nest5SumId);
- val oldIndexId = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream()
+ val oldIndexId = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayouts().stream()
.filter(l -> l.getColOrder().containsAll(indexCol)).findFirst().map(LayoutEntity::getId).orElse(-1L);
// modify expression of cc TEST_KYLIN_FACT.NEST5
val originCC = request.getComputedColumnDescs().stream().filter(c -> ("NEST5").equals(c.getColumnName()))
- .findFirst().orElse(null);
- originCC.setExpression(originCC.getExpression() + "+1");
+ .findFirst();
+ Assert.assertTrue(originCC.isPresent());
+ originCC.get().setExpression(originCC.get().getExpression() + "+1");
modelService.updateDataModelSemantic(getProject(), request);
// new indexes
- val newIndexId = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream()
+ val newIndexId = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayouts().stream()
.filter(l -> l.getColOrder().containsAll(indexCol)).findFirst().map(LayoutEntity::getId).orElse(-2L);
Assert.assertTrue(newIndexId > oldIndexId);
}
@Test
public void testModifyCCChangeType() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
val indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
// create measure
- var request = newSemanticRequest(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
val newMeasure1 = new SimplifiedMeasure();
newMeasure1.setName("NEST5_SUM");
newMeasure1.setExpression("SUM");
@@ -1036,7 +1019,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
newMeasure1.setReturnType("any");
modelService.updateDataModelSemantic(getProject(), request);
// get dim and measure id
- request = newSemanticRequest(modelId);
+ request = newSemanticRequest(BASIC_MODEL);
val transId = request.getColumnIdByColumnName("TEST_KYLIN_FACT.TRANS_ID");
val nest5SumId = request.getSimplifiedMeasures().stream().filter(m -> "NEST5_SUM".equals(m.getName()))
.mapToInt(SimplifiedMeasure::getId).findFirst().orElse(-1);
@@ -1050,32 +1033,32 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
selectRule.jointDims = new Integer[0][0];
newAggregationGroup.setSelectRule(selectRule);
indexPlanService.updateRuleBasedCuboid(getProject(),
- UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(modelId)
- .aggregationGroups(Lists.<NAggregationGroup> newArrayList(newAggregationGroup)).build());
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(Lists.newArrayList(newAggregationGroup)).build());
// old indexes
val indexCol = Arrays.asList(transId, nest5SumId);
- val oldLayoutId = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream()
+ val oldLayoutId = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayouts().stream()
.filter(l -> l.getColOrder().containsAll(indexCol)).findFirst().map(LayoutEntity::getId).orElse(-1L);
// modify expression of cc TEST_KYLIN_FACT.NEST5
val originCC = request.getComputedColumnDescs().stream().filter(c -> ("NEST5").equals(c.getColumnName()))
- .findFirst().orElse(null);
- originCC.setExpression(originCC.getExpression() + "+1");
+ .findFirst();
+ Assert.assertTrue(originCC.isPresent());
+ ComputedColumnDesc cc = originCC.get();
+ cc.setExpression(cc.getExpression() + "+1");
modelService.updateDataModelSemantic(getProject(), request);
// measure NEST5_SUM is reloaded due to return type change
val newNest5SumId = nest5SumId + 1;
val newIndexCol = Arrays.asList(transId, newNest5SumId);
// new layout id
- val newLayoutId = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream()
+ val newLayoutId = indexPlanManager.getIndexPlan(BASIC_MODEL).getAllLayouts().stream()
.filter(l -> l.getColOrder().containsAll(newIndexCol)).findFirst().map(LayoutEntity::getId).orElse(-2L);
Assert.assertTrue(newLayoutId > oldLayoutId);
}
@Test
public void testModifyCCMeasureInvalid() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
- val indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
// create measure
- var request = newSemanticRequest(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
val newMeasure1 = new SimplifiedMeasure();
newMeasure1.setName("NEST5_SUM");
newMeasure1.setExpression("SUM");
@@ -1088,7 +1071,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
newMeasure1.setReturnType("decimal(38, 0)");
modelService.updateDataModelSemantic(getProject(), request);
// get dim and measure id
- request = newSemanticRequest(modelId);
+ request = newSemanticRequest(BASIC_MODEL);
val transId = request.getColumnIdByColumnName("TEST_KYLIN_FACT.TRANS_ID");
val nest5SumId = request.getSimplifiedMeasures().stream().filter(m -> "NEST5_SUM".equals(m.getName()))
.mapToInt(SimplifiedMeasure::getId).findFirst().orElse(-1);
@@ -1102,18 +1085,15 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
selectRule.jointDims = new Integer[0][0];
newAggregationGroup.setSelectRule(selectRule);
indexPlanService.updateRuleBasedCuboid(getProject(),
- UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(modelId)
- .aggregationGroups(Lists.<NAggregationGroup> newArrayList(newAggregationGroup)).build());
- // old indexes
- val indexCol = Arrays.asList(transId, nest5SumId);
- val oldLayoutId = indexPlanManager.getIndexPlan(modelId).getAllLayouts().stream()
- .filter(l -> l.getColOrder().containsAll(indexCol)).findFirst().map(LayoutEntity::getId).orElse(-1L);
+ UpdateRuleBasedCuboidRequest.builder().project(getProject()).modelId(BASIC_MODEL)
+ .aggregationGroups(Lists.newArrayList(newAggregationGroup)).build());
// modify expression of cc TEST_KYLIN_FACT.NEST5
val originCC = request.getComputedColumnDescs().stream().filter(c -> ("NEST5").equals(c.getColumnName()))
- .findFirst().orElse(null);
- originCC.setExpression("'now im a varchar'");
- originCC.setInnerExpression("'now im a varchar'");
- originCC.setDatatype("VARCHAR");
+ .findFirst();
+ Assert.assertTrue(originCC.isPresent());
+ originCC.get().setExpression("'now im a varchar'");
+ originCC.get().setInnerExpression("'now im a varchar'");
+ originCC.get().setDatatype("VARCHAR");
try {
modelService.updateDataModelSemantic(getProject(), request);
} catch (KylinException e) {
@@ -1126,8 +1106,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testModifyCCExistInNestedCC() throws Exception {
// add nested cc
- final String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
- var request = newSemanticRequest(modelId);
+ var request = newSemanticRequest(BASIC_MODEL);
ComputedColumnDesc newCC = new ComputedColumnDesc();
newCC.setColumnName("NEST6");
newCC.setExpression("TEST_KYLIN_FACT.NEST5+1");
@@ -1163,9 +1142,9 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val model = modelService.createModel(request.getProject(), request);
Assert.assertEquals(3, model.getEffectiveMeasures().size());
- Assert.assertThat(
+ Assert.assertEquals(
model.getEffectiveMeasures().values().stream().map(MeasureDesc::getName).collect(Collectors.toList()),
- CoreMatchers.is(Lists.newArrayList("SUM_PRICE", "MAX_COUNT", "COUNT_ALL")));
+ Lists.newArrayList("SUM_PRICE", "MAX_COUNT", "COUNT_ALL"));
}
@Test
@@ -1175,7 +1154,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
+ "by indexes or aggregate groups. Please go to the Data Asset - Model - Index page to view, delete "
+ "referenced aggregate groups and indexes.");
val indePlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- indePlanManager.updateIndexPlan("89af4ee2-2cdb-4b07-b39e-4c29856309aa", cubeBasic -> {
+ indePlanManager.updateIndexPlan(BASIC_MODEL, cubeBasic -> {
val rule = new RuleBasedIndex();
rule.setDimensions(Lists.newArrayList(1, 2, 3, 4, 5, 26));
rule.setMeasures(Lists.newArrayList(100001, 100002, 100003));
@@ -1192,13 +1171,13 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
val originModel = getTestBasicModel();
- modelMgr.updateDataModel(MODEL_ID, model -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> {
val joins = model.getJoinTables();
joins.get(0).getJoin().setType("inner");
});
val cube = dfMgr.getDataflow(originModel.getUuid()).getIndexPlan();
val tableIndexCount = cube.getAllLayouts().stream().filter(l -> l.getIndex().isTableIndex()).count();
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, null, null);
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, null, null);
val executables = getRunningExecutables(getProject(), null);
Assert.assertEquals(1, executables.size());
Assert.assertTrue(((NSparkCubingJob) executables.get(0)).getHandler() instanceof ExecutableAddCuboidHandler);
@@ -1208,21 +1187,20 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testChangePartitionDesc() throws Exception {
+ public void testChangePartitionDesc() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
val originModel = getTestBasicModel();
val cube = dfMgr.getDataflow(originModel.getUuid()).getIndexPlan();
val tableIndexCount = cube.getAllLayouts().stream().filter(l -> l.getIndex().isTableIndex()).count();
- val ids = cube.getAllLayouts().stream().map(LayoutEntity::getId).collect(Collectors.toList());
- modelMgr.updateDataModel(MODEL_ID, model -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> {
val partitionDesc = model.getPartitionDesc();
partitionDesc.setCubePartitionType(PartitionDesc.PartitionType.UPDATE_INSERT);
});
semanticService.handleSemanticUpdate(getProject(), originModel.getUuid(), originModel, null, null);
- val df = dfMgr.getDataflow(MODEL_ID);
+ val df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(0, df.getSegments().size());
Assert.assertEquals(tableIndexCount,
@@ -1250,14 +1228,12 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val cube = dfMgr.getDataflow(originModel.getUuid()).getIndexPlan();
val tableIndexCount = cube.getAllLayouts().stream().filter(l -> l.getIndex().isTableIndex()).count();
- modelMgr.updateDataModel(MODEL_ID, model -> {
- model.setPartitionDesc(null);
- });
+ modelMgr.updateDataModel(BASIC_MODEL, model -> model.setPartitionDesc(null));
semanticService.handleSemanticUpdate(getProject(), originModel.getUuid(), originModel, null, null);
- val executables = getRunningExecutables(getProject(), MODEL_ID);
+ val executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(1, executables.size());
- val df = dfMgr.getDataflow(MODEL_ID);
+ val df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(1, df.getSegments().size());
Assert.assertEquals(tableIndexCount,
@@ -1268,23 +1244,23 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
public void testChangePartitionDesc_NullToOne() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
- modelMgr.updateDataModel(MODEL_ID, model -> model.setPartitionDesc(null));
+ modelMgr.updateDataModel(BASIC_MODEL, model -> model.setPartitionDesc(null));
- val originModel = modelMgr.getDataModelDesc(MODEL_ID);
+ val originModel = modelMgr.getDataModelDesc(BASIC_MODEL);
- modelMgr.updateDataModel(MODEL_ID, model -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> {
val partition = new PartitionDesc();
partition.setPartitionDateColumn("DEFAULT.TEST_KYLIN_FACT.CAL_DT");
partition.setPartitionDateFormat("yyyy-MM-dd");
model.setPartitionDesc(partition);
});
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, "1325347200000", "1388505600000");
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, "1325347200000", "1388505600000");
val executables = getRunningExecutables(getProject(), null);
Assert.assertEquals(1, executables.size());
- val df = dfMgr.getDataflow(MODEL_ID);
+ val df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(1, df.getSegments().size());
@@ -1297,11 +1273,11 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
public void testChangePartitionDesc_NullToOneWithNoDateRange() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
- modelMgr.updateDataModel(MODEL_ID, model -> model.setPartitionDesc(null));
+ modelMgr.updateDataModel(BASIC_MODEL, model -> model.setPartitionDesc(null));
- val originModel = modelMgr.getDataModelDesc(MODEL_ID);
+ val originModel = modelMgr.getDataModelDesc(BASIC_MODEL);
- modelMgr.updateDataModel(MODEL_ID, model -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> {
val partition = new PartitionDesc();
partition.setPartitionDateColumn("DEFAULT.TEST_KYLIN_FACT.CAL_DT");
partition.setPartitionDateFormat("yyyy-MM-dd");
@@ -1312,7 +1288,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val executables = getRunningExecutables(getProject(), null);
Assert.assertEquals(0, executables.size());
- val df = dfMgr.getDataflow(MODEL_ID);
+ val df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(0, df.getSegments().size());
}
@@ -1323,21 +1299,21 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
val originModel = getTestBasicModel();
- modelMgr.updateDataModel(MODEL_ID, model -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> {
val partition = new PartitionDesc();
partition.setPartitionDateColumn("DEFAULT.TEST_KYLIN_FACT.TRANS_ID");
partition.setPartitionDateFormat("yyyy-MM-dd");
model.setPartitionDesc(partition);
});
- var df = dfMgr.getDataflow(MODEL_ID);
+ var df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(1, df.getSegments().size());
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, null, null);
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, null, null);
- val executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ val executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(0, executables.size());
- df = dfMgr.getDataflow(MODEL_ID);
+ df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(0, df.getSegments().size());
}
@@ -1348,21 +1324,21 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
val originModel = getTestBasicModel();
- modelMgr.updateDataModel(MODEL_ID, model -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> {
val partition = new PartitionDesc();
partition.setPartitionDateColumn("DEFAULT.TEST_KYLIN_FACT.TRANS_ID");
partition.setPartitionDateFormat("yyyy-MM-dd");
model.setPartitionDesc(partition);
});
- var df = dfMgr.getDataflow(MODEL_ID);
+ var df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(1, df.getSegments().size());
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, "1325347200000", "1388505600000");
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, "1325347200000", "1388505600000");
val executables = getRunningExecutables(getProject(), null);
Assert.assertEquals(1, executables.size());
- df = dfMgr.getDataflow(MODEL_ID);
+ df = dfMgr.getDataflow(BASIC_MODEL);
Assert.assertEquals(1, df.getSegments().size());
val segment = df.getSegments().get(0);
@@ -1375,34 +1351,34 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
public void testOnlyAddDimensions() throws Exception {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val originModel = getTestBasicModel();
- modelMgr.updateDataModel(MODEL_ID,
+ modelMgr.updateDataModel(BASIC_MODEL,
model -> model.setAllNamedColumns(model.getAllNamedColumns().stream().peek(c -> {
if (!c.isExist()) {
return;
}
c.setStatus(NDataModel.ColumnStatus.DIMENSION);
}).collect(Collectors.toList())));
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, null, null);
- val executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, null, null);
+ val executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(0, executables.size());
}
@Test
- public void testOnlyChangeMeasures() throws Exception {
+ public void testOnlyChangeMeasures() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val indePlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
val originModel = getTestBasicModel();
- modelMgr.updateDataModel(MODEL_ID, model -> model.setAllMeasures(model.getAllMeasures().stream().peek(m -> {
+ modelMgr.updateDataModel(BASIC_MODEL, model -> model.setAllMeasures(model.getAllMeasures().stream().peek(m -> {
if (m.getId() == 100011) {
m.setId(100018);
}
}).collect(Collectors.toList())));
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, null, null);
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, null, null);
var executables = getRunningExecutables(getProject(), null);
Assert.assertEquals(0, executables.size());
- indePlanManager.updateIndexPlan("89af4ee2-2cdb-4b07-b39e-4c29856309aa", copyForWrite -> {
+ indePlanManager.updateIndexPlan(BASIC_MODEL, copyForWrite -> {
val rule = new RuleBasedIndex();
rule.setDimensions(Lists.newArrayList(1, 2, 3, 4, 5, 6));
rule.setMeasures(Lists.newArrayList(100000, 100001));
@@ -1417,20 +1393,20 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
rule.setAggregationGroups(Lists.newArrayList(aggGroup));
copyForWrite.setRuleBasedIndex(rule);
});
- semanticService.handleSemanticUpdate(getProject(), MODEL_ID, originModel, null, null);
+ semanticService.handleSemanticUpdate(getProject(), BASIC_MODEL, originModel, null, null);
- executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(0, executables.size());
- val cube = indePlanManager.getIndexPlan("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ val cube = indePlanManager.getIndexPlan(BASIC_MODEL);
for (LayoutEntity layout : cube.getWhitelistLayouts()) {
- Assert.assertTrue(!layout.getColOrder().contains(100011));
- Assert.assertTrue(!layout.getIndex().getMeasures().contains(100011));
+ Assert.assertFalse(layout.getColOrder().contains(100011));
+ Assert.assertFalse(layout.getIndex().getMeasures().contains(100011));
}
}
@Test
- public void testOnlyChangeMeasuresWithRule() throws Exception {
+ public void testOnlyChangeMeasuresWithRule() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
val indePlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
val originModel = getTestInnerModel();
@@ -1443,13 +1419,13 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
semanticService.handleSemanticUpdate(getProject(), originModel.getUuid(), originModel, null, null);
- val cube = indePlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val cube = indePlanManager.getIndexPlan(INNER_MODEL);
for (LayoutEntity layout : cube.getWhitelistLayouts()) {
- Assert.assertTrue(!layout.getColOrder().contains(100017));
- Assert.assertTrue(!layout.getIndex().getMeasures().contains(100017));
+ Assert.assertFalse(layout.getColOrder().contains(100017));
+ Assert.assertFalse(layout.getIndex().getMeasures().contains(100017));
}
val newRule = cube.getRuleBasedIndex();
- Assert.assertTrue(!newRule.getMeasures().contains(100017));
+ Assert.assertFalse(newRule.getMeasures().contains(100017));
}
@Test
@@ -1483,18 +1459,18 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(1, executables.size());
Assert.assertTrue(((NSparkCubingJob) executables.get(0)).getHandler() instanceof ExecutableAddCuboidHandler);
- val cube = indePlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val cube = indePlanManager.getIndexPlan(INNER_MODEL);
for (LayoutEntity layout : cube.getWhitelistLayouts()) {
- Assert.assertTrue(!layout.getColOrder().contains(100011));
- Assert.assertTrue(!layout.getIndex().getMeasures().contains(100011));
+ Assert.assertFalse(layout.getColOrder().contains(100011));
+ Assert.assertFalse(layout.getIndex().getMeasures().contains(100011));
}
}
@Test
- public void testOnlyRuleChanged() throws Exception {
+ public void testOnlyRuleChanged() {
val indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
val dfMgr = NDataflowManager.getInstance(getTestConfig(), getProject());
- val df = dfMgr.getDataflow("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val df = dfMgr.getDataflow(INNER_MODEL);
val originSegLayoutSize = df.getSegments().get(0).getLayoutsMap().size();
NDataflowUpdate update = new NDataflowUpdate(df.getUuid());
val cube = df.getIndexPlan();
@@ -1516,7 +1492,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
semanticService.handleIndexPlanUpdateRule(getProject(), df.getModel().getUuid(), cube.getRuleBasedIndex(),
newCube.getRuleBasedIndex(), false);
- val executables = getRunningExecutables(getProject(), "741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val executables = getRunningExecutables(getProject(), INNER_MODEL);
Assert.assertEquals(1, executables.size());
Assert.assertTrue(((NSparkCubingJob) executables.get(0)).getHandler() instanceof ExecutableAddCuboidHandler);
@@ -1525,11 +1501,11 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testOnlyRemoveColumns_removeToBeDeletedIndex() throws Exception {
+ public void testOnlyRemoveColumns_removeToBeDeletedIndex() {
val modelManager = NDataModelManager.getInstance(getTestConfig(), getProject());
val indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- val indexPlan = indexPlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val indexPlan = indexPlanManager.getIndexPlan(INNER_MODEL);
val originModel = getTestInnerModel();
NIndexPlanManager.getInstance(getTestConfig(), getProject()).updateIndexPlan(indexPlan.getUuid(),
@@ -1545,7 +1521,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
model.getAllNamedColumns().stream().filter(m -> m.getId() != 25).collect(Collectors.toList())));
NDataflowManager dataflowManager = NDataflowManager.getInstance(getTestConfig(), getProject());
- NDataflow dataflow = dataflowManager.getDataflow("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ NDataflow dataflow = dataflowManager.getDataflow(INNER_MODEL);
NIndexPlanManager.getInstance(getTestConfig(), getProject()).updateIndexPlan(dataflow.getUuid(),
copyForWrite -> {
val toBeDeletedSet = copyForWrite.getIndexes().stream().map(IndexEntity::getLayouts)
@@ -1553,8 +1529,8 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
.collect(Collectors.toSet());
copyForWrite.markIndexesToBeDeleted(dataflow.getUuid(), toBeDeletedSet);
});
- Assert.assertTrue(CollectionUtils.isNotEmpty(
- indexPlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a").getToBeDeletedIndexes()));
+ Assert.assertTrue(
+ CollectionUtils.isNotEmpty(indexPlanManager.getIndexPlan(INNER_MODEL).getToBeDeletedIndexes()));
indexPlanManager.updateIndexPlan(indexPlan.getUuid(), k -> {
val newDim = k.getRuleBasedIndex().getDimensions().stream().filter(x -> x != 25)
.collect(Collectors.toList());
@@ -1567,16 +1543,15 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
k.getRuleBasedIndex().setAggregationGroups(aggs);
});
semanticService.handleSemanticUpdate(getProject(), indexPlan.getUuid(), originModel, null, null);
- Assert.assertTrue(CollectionUtils.isEmpty(
- indexPlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a").getToBeDeletedIndexes()));
+ Assert.assertTrue(CollectionUtils.isEmpty(indexPlanManager.getIndexPlan(INNER_MODEL).getToBeDeletedIndexes()));
}
@Test
- public void testOnlyRemoveMeasures() throws Exception {
+ public void testOnlyRemoveMeasures() {
val modelManager = NDataModelManager.getInstance(getTestConfig(), getProject());
val indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- val indexPlan = indexPlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val indexPlan = indexPlanManager.getIndexPlan(INNER_MODEL);
val originModel = getTestInnerModel();
indexPlanManager.updateIndexPlan(indexPlan.getId(), k -> {
@@ -1596,7 +1571,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
.collect(Collectors.toList())));
semanticService.handleSemanticUpdate(getProject(), indexPlan.getUuid(), originModel, null, null);
- val executables = getRunningExecutables(getProject(), "741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val executables = getRunningExecutables(getProject(), INNER_MODEL);
Assert.assertEquals(1, executables.size());
val newCube = indexPlanManager.getIndexPlan(indexPlan.getUuid());
@@ -1605,11 +1580,11 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testSetBlackListLayout() throws Exception {
+ public void testSetBlackListLayout() {
val indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
- val indexPlan = indexPlanManager.getIndexPlan("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val indexPlan = indexPlanManager.getIndexPlan(INNER_MODEL);
val dataflowManager = NDataflowManager.getInstance(getTestConfig(), getProject());
- val dataflow = dataflowManager.getDataflow("741ca86a-1f13-46da-a59f-95fb68615e3a");
+ val dataflow = dataflowManager.getDataflow(INNER_MODEL);
val dfUpdate = new NDataflowUpdate(dataflow.getUuid());
List<NDataLayout> layouts = Lists.newArrayList();
@@ -1652,14 +1627,12 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
private NDataModel getTestInnerModel() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
- val model = modelMgr.getDataModelDesc("741ca86a-1f13-46da-a59f-95fb68615e3a");
- return model;
+ return modelMgr.getDataModelDesc(INNER_MODEL);
}
private NDataModel getTestBasicModel() {
val modelMgr = NDataModelManager.getInstance(getTestConfig(), getProject());
- val model = modelMgr.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
- return model;
+ return modelMgr.getDataModelDesc(BASIC_MODEL);
}
private ModelRequest changeAlias(ModelRequest request, String old, String newAlias) throws IOException {
@@ -1689,7 +1662,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
}
private ModelRequest newSemanticRequest() throws Exception {
- return newSemanticRequest("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ return newSemanticRequest(BASIC_MODEL);
}
private ModelRequest newSemanticRequest(String modelId) throws Exception {
@@ -1717,7 +1690,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
private NDataModel getTestModel() {
val modelMgr = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
- return modelMgr.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ return modelMgr.getDataModelDesc(BASIC_MODEL);
}
@Test
@@ -1767,7 +1740,7 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
@Test
public void testUpdateDataModelParatitionDesc() {
val modelMgr = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), getProject());
- var model = modelMgr.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ var model = modelMgr.getDataModelDesc(BASIC_MODEL);
Assert.assertNotNull(model.getPartitionDesc());
ModelParatitionDescRequest modelParatitionDescRequest = new ModelParatitionDescRequest();
modelParatitionDescRequest.setStart("0");
@@ -1775,33 +1748,38 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
modelParatitionDescRequest.setPartitionDesc(null);
PartitionDesc partitionDesc = model.getPartitionDesc();
- var executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ var executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(0, executables.size());
modelService.updateModelPartitionColumn(getProject(), model.getAlias(), modelParatitionDescRequest);
- model = modelMgr.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ model = modelMgr.getDataModelDesc(BASIC_MODEL);
Assert.assertNull(model.getPartitionDesc());
- executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(1, executables.size());
modelParatitionDescRequest.setPartitionDesc(partitionDesc);
deleteJobByForce(executables.get(0));
modelService.updateModelPartitionColumn(getProject(), model.getAlias(), modelParatitionDescRequest);
- model = modelMgr.getDataModelDesc("89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ model = modelMgr.getDataModelDesc(BASIC_MODEL);
Assert.assertEquals(partitionDesc, model.getPartitionDesc());
- executables = getRunningExecutables(getProject(), "89af4ee2-2cdb-4b07-b39e-4c29856309aa");
+ executables = getRunningExecutables(getProject(), BASIC_MODEL);
Assert.assertEquals(1, executables.size());
}
@Test
public void testModelSemanticUpdateNoBlackListLayoutRestore() throws Exception {
- String modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ String modelId = BASIC_MODEL;
val newRule = new RuleBasedIndex();
newRule.setDimensions(Arrays.asList(14, 15, 16));
- val group1 = JsonUtil.readValue("{\n" + " \"includes\": [14,15,16],\n" + " \"select_rule\": {\n"
- + " \"hierarchy_dims\": [],\n" + " \"mandatory_dims\": [],\n"
- + " \"joint_dims\": []\n" + " }\n" + "}", NAggregationGroup.class);
+ val group1 = JsonUtil.readValue("{\n" //
+ + " \"includes\": [14,15,16],\n" //
+ + " \"select_rule\": {\n" //
+ + " \"hierarchy_dims\": [],\n" //
+ + " \"mandatory_dims\": [],\n" //
+ + " \"joint_dims\": []\n" //
+ + " }\n" //
+ + "}", NAggregationGroup.class);
newRule.setAggregationGroups(Lists.newArrayList(group1));
group1.setMeasures(new Integer[] { 100000, 100008 });
val indexManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
@@ -1829,10 +1807,8 @@ public class ModelServiceSemanticUpdateTest extends NLocalFileMetadataTestCase {
request.getSimplifiedMeasures().add(newMeasure1);
newMeasure1.setReturnType("decimal(38, 0)");
modelService.updateDataModelSemantic(getProject(), request);
- Assert.assertThat(indexPlanManager.getIndexPlan(modelId).getRuleBasedIndex().getLayoutBlackList().size(),
- is(7));
- Assert.assertThat(indexPlanManager.getIndexPlan(modelId).getRuleBasedIndex().genCuboidLayouts().size(), is(0));
-
+ Assert.assertEquals(7, indexPlanManager.getIndexPlan(modelId).getRuleBasedIndex().getLayoutBlackList().size());
+ Assert.assertTrue(indexPlanManager.getIndexPlan(modelId).getRuleBasedIndex().genCuboidLayouts().isEmpty());
}
protected List<AbstractExecutable> getRunningExecutables(String project, String model) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 8640e901d9..66ab4a09f1 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -219,7 +219,6 @@ import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.guava30.shaded.common.primitives.Longs;
-//import io.kyligence.kap.clickhouse.MockSecondStorage;
import io.kyligence.kap.secondstorage.SecondStorageNodeHelper;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
import io.kyligence.kap.secondstorage.config.Node;
@@ -1865,10 +1864,10 @@ public class ModelServiceTest extends SourceTestCase {
modelService.createModel(modelRequest.getProject(), modelRequest);
}
- private List<NonEquiJoinCondition.SimplifiedNonEquiJoinCondition> genNonEquiJoinCond() {
- NonEquiJoinCondition.SimplifiedNonEquiJoinCondition join1 = new NonEquiJoinCondition.SimplifiedNonEquiJoinCondition(
+ private List<NonEquiJoinCondition.SimplifiedJoinCondition> genNonEquiJoinCond() {
+ NonEquiJoinCondition.SimplifiedJoinCondition join1 = new NonEquiJoinCondition.SimplifiedJoinCondition(
"TEST_KYLIN_FACT.SELLER_ID", "TEST_ORDER.TEST_EXTENDED_COLUMN", SqlKind.GREATER_THAN_OR_EQUAL);
- NonEquiJoinCondition.SimplifiedNonEquiJoinCondition join2 = new NonEquiJoinCondition.SimplifiedNonEquiJoinCondition(
+ NonEquiJoinCondition.SimplifiedJoinCondition join2 = new NonEquiJoinCondition.SimplifiedJoinCondition(
"TEST_KYLIN_FACT.SELLER_ID", "TEST_ORDER.BUYER_ID", SqlKind.LESS_THAN);
return Arrays.asList(join1, join2);
}
@@ -3476,7 +3475,7 @@ public class ModelServiceTest extends SourceTestCase {
okModel.setFilterCondition("TEST_KYLIN_FACT.SELLER_ID > 0");
ModelRequest okModelRequest = new ModelRequest(okModel);
okModelRequest.setProject(project);
- when(semanticService.convertToDataModel(okModelRequest)).thenReturn(okModel);
+ doReturn(okModel).when(semanticService).convertToDataModel(okModelRequest);
okModelRequest.setPartitionDesc(null);
modelService.checkBeforeModelSave(okModelRequest);
}
@@ -3550,7 +3549,8 @@ public class ModelServiceTest extends SourceTestCase {
+ "and TEST_KYLIN_FACT.order_id < 100 and DEAL_AMOUNT > 123";
model.setFilterCondition(originSql);
modelService.massageModelFilterCondition(model);
- Assert.assertEquals("(((((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) AND (`TEST_KYLIN_FACT`.`TRANS_ID` > 0)) "
+ Assert.assertEquals(
+ "(((((`TEST_KYLIN_FACT`.`TRANS_ID` = 0) AND (`TEST_KYLIN_FACT`.`TRANS_ID` > 0)) "
+ "AND (`TEST_KYLIN_FACT`.`TRANS_ID` > 0)) AND (`TEST_KYLIN_FACT`.`ORDER_ID` < 100)) "
+ "AND ((`TEST_KYLIN_FACT`.`PRICE` * `TEST_KYLIN_FACT`.`ITEM_COUNT`) > 123))",
model.getFilterCondition());
@@ -5740,16 +5740,15 @@ public class ModelServiceTest extends SourceTestCase {
@Test
public void testInitModel() throws IOException {
- String modelRequest =
- "{\"uuid\":null,\"name\":\"sum_lc_null_val_test_clone\",\"owner\":\"ADMIN\",\"project\":\"sum_lc\",\"description\":null,"
- + "\"alias\":\"sum_lc_null_val_test_clone\",\"fact_table\":\"SSB.SUM_LC_NULL_TBL\",\"join_tables\":[],"
- + "\"simplified_dimensions\":[{\"id\":0,\"name\":\"PART_COL\",\"column\":\"SUM_LC_NULL_TBL.PART_COL\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"date\",\"simple\":null,\"datatype\":\"date\"},"
- + "{\"id\":1,\"name\":\"SUM_DATE1\",\"column\":\"SUM_LC_NULL_TBL.SUM_DATE1\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"varchar(1024)\",\"simple\":null,\"datatype\":\"varchar(1024)\"},{\"id\":2,\"name\":\"ACCOUNT1\",\"column\":\"SUM_LC_NULL_TBL.ACCOUNT1\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardin [...]
- + "{\"id\":6,\"name\":\"ACCOUNT2\",\"column\":\"SUM_LC_NULL_TBL.ACCOUNT2\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"varchar(52)\",\"simple\":null,\"datatype\":\"varchar(52)\"}],"
- + "\"simplified_measures\":[{\"id\":100000,\"expression\":\"COUNT\",\"name\":\"COUNT_ALL\",\"return_type\":\"bigint\",\"parameter_value\":[{\"type\":\"constant\",\"value\":\"1\"}],\"converted_columns\":[],\"column\":null,\"comment\":null},{\"id\":100001,\"expression\":\"SUM_LC\",\"name\":\"sumlc_double_null\",\"return_type\":\"double\",\"parameter_value\":[{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.DATA_NULL\"},{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TB [...]
- + "\"converted_columns\":[],\"column\":null,\"comment\":\"\"},{\"id\":100002,\"expression\":\"SUM_LC\",\"name\":\"sumlc_decimal_null\",\"return_type\":\"decimal(20,6)\",\"parameter_value\":[{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.DATA_DECIMAL\"},{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.SUM_DATE1\"}],\"converted_columns\":[],\"column\":null,\"comment\":\"\"},{\"name\":\"abc\",\"expression\":\"SUM_LC\",\"return_type\":\"\",\"comment\":\"\",\"paramet [...]
- + "\"computed_columns\":[],\"last_modified\":1668402813791,\"filter_condition\":\"\",\"partition_desc\":null,\"multi_partition_desc\":null,\"management_type\":\"MODEL_BASED\",\"with_second_storage\":false,\"second_storage_size\":0,\"canvas\":{\"coordinate\":{\"SUM_LC_NULL_TBL\":{\"x\":462.44444105360253,\"y\":108.66667005750864,\"width\":200,\"height\":486.66666666666663,\"isSpread\":true}},\"zoom\":9,\"marginClient\":{\"left\":0,\"top\":0}},\"available_indexes_co [...]
- + "{\"column\":\"SUM_LC_NULL_TBL.DATA_NULL\",\"name\":\"DATA_NULL\",\"datatype\":\"double\"},{\"column\":\"SUM_LC_NULL_TBL.DATA_DECIMAL\",\"name\":\"DATA_DECIMAL\",\"datatype\":\"decimal(10,6)\"}]}";
+ String modelRequest = "{\"uuid\":null,\"name\":\"sum_lc_null_val_test_clone\",\"owner\":\"ADMIN\",\"project\":\"sum_lc\",\"description\":null,"
+ + "\"alias\":\"sum_lc_null_val_test_clone\",\"fact_table\":\"SSB.SUM_LC_NULL_TBL\",\"join_tables\":[],"
+ + "\"simplified_dimensions\":[{\"id\":0,\"name\":\"PART_COL\",\"column\":\"SUM_LC_NULL_TBL.PART_COL\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"date\",\"simple\":null,\"datatype\":\"date\"},"
+ + "{\"id\":1,\"name\":\"SUM_DATE1\",\"column\":\"SUM_LC_NULL_TBL.SUM_DATE1\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"varchar(1024)\",\"simple\":null,\"datatype\":\"varchar(1024)\"},{\"id\":2,\"name\":\"ACCOUNT1\",\"column\":\"SUM_LC_NULL_TBL.ACCOUNT1\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\": [...]
+ + "{\"id\":6,\"name\":\"ACCOUNT2\",\"column\":\"SUM_LC_NULL_TBL.ACCOUNT2\",\"status\":\"DIMENSION\",\"excluded\":false,\"cardinality\":null,\"min_value\":null,\"max_value\":null,\"max_length_value\":null,\"min_length_value\":null,\"null_count\":null,\"comment\":null,\"type\":\"varchar(52)\",\"simple\":null,\"datatype\":\"varchar(52)\"}],"
+ + "\"simplified_measures\":[{\"id\":100000,\"expression\":\"COUNT\",\"name\":\"COUNT_ALL\",\"return_type\":\"bigint\",\"parameter_value\":[{\"type\":\"constant\",\"value\":\"1\"}],\"converted_columns\":[],\"column\":null,\"comment\":null},{\"id\":100001,\"expression\":\"SUM_LC\",\"name\":\"sumlc_double_null\",\"return_type\":\"double\",\"parameter_value\":[{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.DATA_NULL\"},{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.SUM_DA [...]
+ + "\"converted_columns\":[],\"column\":null,\"comment\":\"\"},{\"id\":100002,\"expression\":\"SUM_LC\",\"name\":\"sumlc_decimal_null\",\"return_type\":\"decimal(20,6)\",\"parameter_value\":[{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.DATA_DECIMAL\"},{\"type\":\"column\",\"value\":\"SUM_LC_NULL_TBL.SUM_DATE1\"}],\"converted_columns\":[],\"column\":null,\"comment\":\"\"},{\"name\":\"abc\",\"expression\":\"SUM_LC\",\"return_type\":\"\",\"comment\":\"\",\"parameter_value [...]
+ + "\"computed_columns\":[],\"last_modified\":1668402813791,\"filter_condition\":\"\",\"partition_desc\":null,\"multi_partition_desc\":null,\"management_type\":\"MODEL_BASED\",\"with_second_storage\":false,\"second_storage_size\":0,\"canvas\":{\"coordinate\":{\"SUM_LC_NULL_TBL\":{\"x\":462.44444105360253,\"y\":108.66667005750864,\"width\":200,\"height\":486.66666666666663,\"isSpread\":true}},\"zoom\":9,\"marginClient\":{\"left\":0,\"top\":0}},\"available_indexes_count\":0, [...]
+ + "{\"column\":\"SUM_LC_NULL_TBL.DATA_NULL\",\"name\":\"DATA_NULL\",\"datatype\":\"double\"},{\"column\":\"SUM_LC_NULL_TBL.DATA_DECIMAL\",\"name\":\"DATA_DECIMAL\",\"datatype\":\"decimal(10,6)\"}]}";
ModelRequest request = JsonUtil.readValue(modelRequest, ModelRequest.class);
Assert.assertThrows(KylinException.class, () -> modelService.checkBeforeModelSave(request));
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
index abb8abc371..865fe593a2 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceColumnNameTest.java
@@ -89,7 +89,7 @@ public class ModelTdsServiceColumnNameTest extends SourceTestCase {
ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "accessService", accessService);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
- ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(tdsService, "accessService", accessService);
ReflectionTestUtils.setField(tdsService, "userGroupService", userGroupService);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
index a89749b91c..cd39188c1e 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
@@ -36,6 +36,11 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.base.Charsets;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.io.CharStreams;
import org.apache.kylin.junit.rule.TransactionExceptedException;
import org.apache.kylin.metadata.acl.AclTCR;
import org.apache.kylin.metadata.acl.AclTCRManager;
@@ -66,12 +71,6 @@ import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.base.Charsets;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.io.CharStreams;
-
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -117,7 +116,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelService, "accessService", accessService);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
- ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(tdsService, "accessService", accessService);
ReflectionTestUtils.setField(tdsService, "userGroupService", userGroupService);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index baffa9d374..04b3f94fe8 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -193,7 +193,7 @@ public class TableServiceTest extends CSVSourceTestCase {
projectInstance.getOwner(), projectInstance.getDescription(), overrideKylinProps);
projectManager.updateProject(projectInstance, projectInstanceUpdate.getName(),
projectInstanceUpdate.getDescription(), projectInstanceUpdate.getOverrideKylinProps());
- Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin"));
+ Mockito.doReturn(Collections.singletonList("admin")).when(userService).listSuperAdminUsers();
try {
new JdbcRawRecStore(getTestConfig());
} catch (Exception e) {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
index ac68e861ec..d3cfda0c2c 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
@@ -52,18 +52,17 @@ import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.NonEquiJoinCondition;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.query.util.RexToTblColRefTranslator;
import org.apache.kylin.query.util.ICutContextStrategy;
+import org.apache.kylin.query.util.RexToTblColRefTranslator;
import org.apache.kylin.query.util.RexUtils;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
private OLAPContext context;
@@ -73,18 +72,17 @@ public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
private boolean isPreCalJoin = true;
private boolean aboveContextPreCalcJoin = false;
- // record left input size before rewrite for runtime join expression parseing
+ // record left input size before rewrite for runtime join expression parsing
private int leftInputSizeBeforeRewrite = -1;
- private boolean isQueryNonEquiJoinModelEnabled;
+ private final boolean isScd2Rel;
public KapNonEquiJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition,
- Set<CorrelationId> variablesSet, JoinRelType joinType, boolean isQueryNonEquiJoinModelEnabled)
- throws InvalidRelException {
+ Set<CorrelationId> variablesSet, JoinRelType joinType, boolean isScd2Rel) throws InvalidRelException {
super(cluster, traits, left, right, condition, variablesSet, joinType);
leftInputSizeBeforeRewrite = left.getRowType().getFieldList().size();
rowType = getRowType();
- this.isQueryNonEquiJoinModelEnabled = isQueryNonEquiJoinModelEnabled;
+ this.isScd2Rel = isScd2Rel;
}
@Override
@@ -106,10 +104,7 @@ public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
private void allocateContext(ContextVisitorState leftState, ContextVisitorState rightState,
OLAPContextImplementor olapContextImplementor) {
- // if query on non-equi-join model is not enabled
- // allocate context like runtime join directly
- // inner join shouldnt run here with run-time join
- if (getJoinType() == JoinRelType.LEFT && !isQueryNonEquiJoinModelEnabled) {
+ if (!isScd2Rel) {
if (leftState.hasFreeTable()) {
olapContextImplementor.allocateContext((KapRel) left, this);
leftState.setHasFreeTable(false);
@@ -121,7 +116,7 @@ public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
return;
}
- if (getJoinType() == JoinRelType.LEFT && rightState.hasFreeTable() && rightState.hasFilter()) {
+ if (rightState.hasFreeTable() && rightState.hasFilter()) {
olapContextImplementor.allocateContext((KapRel) right, this);
rightState.setHasFreeTable(false);
}
@@ -440,7 +435,7 @@ public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
JoinRelType joinType, boolean semiJoinDone) {
try {
return new KapNonEquiJoinRel(getCluster(), traitSet, left, right, condition, variablesSet, joinType,
- isQueryNonEquiJoinModelEnabled);
+ isScd2Rel);
} catch (InvalidRelException e) {
// Semantic error not possible. Must be a bug. Convert to
// internal error.
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 3757d717d8..b2aeac6995 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -54,16 +54,22 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.NonEquiJoinCondition;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.util.scd2.Scd2Simplifier;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
@@ -77,11 +83,6 @@ import org.codehaus.commons.compiler.CompileException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
public class PushDownUtil {
private static final Logger logger = LoggerFactory.getLogger("query");
@@ -207,7 +208,8 @@ public class PushDownUtil {
.map(c -> c.getClass().getCanonicalName()).collect(Collectors.joining(",")));
}
for (IPushDownConverter converter : pushDownConverters) {
- QueryInterruptChecker.checkThreadInterrupted("Interrupted sql transformation at the stage of " + converter.getClass(),
+ QueryInterruptChecker.checkThreadInterrupted(
+ "Interrupted sql transformation at the stage of " + converter.getClass(),
"Current step: Massage push-down sql. ");
sql = converter.convert(sql, queryParams.getProject(), queryParams.getDefaultSchema());
}
@@ -297,8 +299,7 @@ public class PushDownUtil {
for (JoinTableDesc lookupDesc : model.getJoinTables()) {
JoinDesc join = lookupDesc.getJoin();
TableRef dimTable = lookupDesc.getTableRef();
- if (join == null || org.apache.commons.lang3.StringUtils.isEmpty(join.getType())
- || dimTableCache.contains(dimTable)) {
+ if (join == null || StringUtils.isEmpty(join.getType()) || dimTableCache.contains(dimTable)) {
continue;
}
@@ -313,13 +314,16 @@ public class PushDownUtil {
.append(" as ").append(StringHelper.doubleQuote(dimTable.getAlias())) //
.append(sep).append("ON ");
- if (pk.length == 0 && join.getNonEquiJoinCondition() != null) {
- sql.append(join.getNonEquiJoinCondition().getExpr());
- dimTableCache.add(dimTable);
- } else {
- sql.append(concatEqualJoinCondition(pk, fk, sep));
- dimTableCache.add(dimTable);
+ sql.append(concatEqualJoinCondition(pk, fk, sep));
+ if (join.getNonEquiJoinCondition() != null) {
+ NonEquiJoinCondition[] operands = join.getNonEquiJoinCondition().getOperands();
+ for (NonEquiJoinCondition operand : operands) {
+ sql.append(" AND ");
+ String cond = Scd2Simplifier.INSTANCE.simplifySCD2ChildCond(operand, join).displaySql();
+ sql.append(cond).append(sep);
+ }
}
+ dimTableCache.add(dimTable);
}
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapJoinRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapJoinRule.java
index a21c2dbc97..e31c5b2164 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapJoinRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapJoinRule.java
@@ -19,8 +19,11 @@
package io.kyligence.kap.query.optrule;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.calcite.plan.Convention;
import org.apache.calcite.plan.RelOptCluster;
@@ -34,38 +37,60 @@ import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.util.ImmutableIntList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.relnode.KapNonEquiJoinRel;
-import org.apache.kylin.query.relnode.KapRel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.query.relnode.OLAPRel;
public class KapJoinRule extends ConverterRule {
- private static final Logger logger = LoggerFactory.getLogger(KapJoinRule.class);
+
+ private static final Set<Set<SqlOperator>> ALLOWED_PAIRS = ImmutableSet.of(
+ ImmutableSet.of(SqlStdOperatorTable.GREATER_THAN, SqlStdOperatorTable.LESS_THAN_OR_EQUAL),
+ ImmutableSet.of(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, SqlStdOperatorTable.LESS_THAN_OR_EQUAL),
+ ImmutableSet.of(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, SqlStdOperatorTable.LESS_THAN));
+
+ private SqlOperator inverse(SqlOperator operator) {
+ if (operator.equals(SqlStdOperatorTable.GREATER_THAN)) {
+ return SqlStdOperatorTable.LESS_THAN;
+ } else if (operator.equals(SqlStdOperatorTable.LESS_THAN)) {
+ return SqlStdOperatorTable.GREATER_THAN;
+ } else if (operator.equals(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL)) {
+ return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
+ } else if (operator.equals(SqlStdOperatorTable.LESS_THAN_OR_EQUAL)) {
+ return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL;
+ }
+ return null;
+ }
public static final ConverterRule INSTANCE = new KapJoinRule();
public static final ConverterRule NON_EQUI_INSTANCE = new KapJoinRule(true, false);
public static final ConverterRule EQUAL_NULL_SAFE_INSTANT = new KapJoinRule(false, true);
+ private static final ImmutableSet<SqlKind> SCD2_KINDS = ImmutableSet.of(SqlKind.GREATER_THAN, SqlKind.LESS_THAN,
+ SqlKind.GREATER_THAN_OR_EQUAL, SqlKind.LESS_THAN_OR_EQUAL);
- private boolean isQueryNonEquiJoinModelEnabled;
- private boolean joinCondEqualNullSafe;
+ private final boolean isScd2Enabled;
+ private final boolean joinCondEqualNullSafe;
public KapJoinRule() {
this(false, false);
}
- public KapJoinRule(boolean isQueryNonEquiJoinModelEnabled, boolean joinCondEqualNullSafe) {
- super(LogicalJoin.class, Convention.NONE, KapRel.CONVENTION, "KapJoinRule");
- this.isQueryNonEquiJoinModelEnabled = isQueryNonEquiJoinModelEnabled;
+ public KapJoinRule(boolean isScd2Enabled, boolean joinCondEqualNullSafe) {
+ super(LogicalJoin.class, Convention.NONE, OLAPRel.CONVENTION, "KapJoinRule");
+ this.isScd2Enabled = isScd2Enabled;
this.joinCondEqualNullSafe = joinCondEqualNullSafe;
}
@@ -75,11 +100,11 @@ public class KapJoinRule extends ConverterRule {
RelNode left = join.getInput(0);
RelNode right = join.getInput(1);
- RelTraitSet traitSet = join.getTraitSet().replace(KapRel.CONVENTION);
- left = left instanceof HepRelVertex ? left : convert(left, left.getTraitSet().replace(KapRel.CONVENTION));
- right = right instanceof HepRelVertex ? right : convert(right, right.getTraitSet().replace(KapRel.CONVENTION));
+ RelTraitSet traitSet = join.getTraitSet().replace(OLAPRel.CONVENTION);
+ left = left instanceof HepRelVertex ? left : convert(left, left.getTraitSet().replace(OLAPRel.CONVENTION));
+ right = right instanceof HepRelVertex ? right : convert(right, right.getTraitSet().replace(OLAPRel.CONVENTION));
- final JoinInfo info = JoinInfo.of(left, right, join.getCondition());
+ JoinInfo info = JoinInfo.of(left, right, join.getCondition());
// handle powerbi inner join
Join tmpJoin = transformJoinCondition(join, info, traitSet, left, right);
@@ -87,60 +112,83 @@ public class KapJoinRule extends ConverterRule {
return tmpJoin;
}
- RelNode newRel;
+ // usually non-equiv-join can not be converted to equiv-join + filter
+ // keep KapNonEquiJoinRel and cut to small OlapContexts in the class of KapNonEquiJoinRel
try {
- if (isNonEquiJoinRelRule(info.isEqui(), join.getJoinType())
- || isNotDistinctFrom(left, right, join.getCondition())) {
- try {
- return new KapNonEquiJoinRel(join.getCluster(), traitSet, left, right, join.getCondition(),
- join.getVariablesSet(), join.getJoinType(), isQueryNonEquiJoinModelEnabled);
- } catch (InvalidRelException e) {
- throw new IllegalStateException(e);
+ RexBuilder rexBuilder = join.getCluster().getRexBuilder();
+ RexNode cnfCondition = RexUtil.toCnf(rexBuilder, join.getCondition());
+ info = JoinInfo.of(left, right, cnfCondition);
+ if (!info.isEqui()) {
+ List<RexInputRef> scd2Refs = Lists.newArrayList();
+ boolean isScd2Rel = isScd2Enabled && isScd2JoinCondition(info, join, scd2Refs);
+ if (join.getJoinType() == JoinRelType.INNER && !isScd2Rel
+ && hasEqualJoinPart(left, right, join.getCondition())) {
+ KapJoinRel joinRel = new KapJoinRel(join.getCluster(), traitSet, left, right,
+ info.getEquiCondition(left, right, rexBuilder), info.leftKeys, info.rightKeys,
+ join.getVariablesSet(), join.getJoinType());
+ joinRel.setJoinCondEqualNullSafe(joinCondEqualNullSafe);
+ RexNode rexNode = info.getRemaining(rexBuilder);
+ return rexNode.isAlwaysTrue() ? joinRel
+ : new KapFilterRel(join.getCluster(), joinRel.getTraitSet(), joinRel, rexNode);
}
+
+ // cnf is better, but conflict with transformJoinCondition, need optimize
+ RexNode joinCondition = normalizeCondition(rexBuilder, join.getCondition(), scd2Refs);
+ return new KapNonEquiJoinRel(join.getCluster(), traitSet, left, right, joinCondition,
+ join.getVariablesSet(), join.getJoinType(), isScd2Rel);
} else {
- newRel = new KapJoinRel(join.getCluster(), traitSet, left, right,
- info.getEquiCondition(left, right, join.getCluster().getRexBuilder()), info.leftKeys,
- info.rightKeys, join.getVariablesSet(), join.getJoinType());
- ((KapJoinRel) newRel).setJoinCondEqualNullSafe(joinCondEqualNullSafe);
- }
- if (!info.isEqui()) {
- newRel = new KapFilterRel(join.getCluster(), newRel.getTraitSet(), newRel,
- info.getRemaining(join.getCluster().getRexBuilder()));
+ KapJoinRel joinRel = new KapJoinRel(join.getCluster(), traitSet, left, right,
+ info.getEquiCondition(left, right, rexBuilder), info.leftKeys, info.rightKeys,
+ join.getVariablesSet(), join.getJoinType());
+ joinRel.setJoinCondEqualNullSafe(joinCondEqualNullSafe);
+ return joinRel;
}
} catch (InvalidRelException e) {
- // Semantic error not possible. Must be a bug. Convert to internal error.
throw new AssertionError(e);
- // LOGGER.fine(e.toString());
}
- return newRel;
}
- private boolean isNonEquiJoinRelRule(boolean isEqui, JoinRelType joinRelType) {
-
- if (isEqui || (joinRelType == JoinRelType.INNER && !isQueryNonEquiJoinModelEnabled)) {
- return false;
+ private RexNode normalizeCondition(RexBuilder rexBuilder, RexNode cnfCondition, List<RexInputRef> scd2Refs) {
+ if (scd2Refs.isEmpty() || !(cnfCondition instanceof RexCall)) {
+ return cnfCondition;
}
- return true;
+ List<RexNode> newNodes = Lists.newArrayList();
+ List<RexNode> oriNodes = ((RexCall) cnfCondition).getOperands();
+ for (RexNode rexNode : oriNodes) {
+ RexNode invert = rexNode;
+ if (SCD2_KINDS.contains(rexNode.getKind())) {
+ RexNode left = ((RexCall) invert).getOperands().get(0);
+ RexInputRef rexInputRef = extractInputRef(left);
+ if (!scd2Refs.contains(rexInputRef)) {
+ invert = RexUtil.invert(rexBuilder, (RexCall) rexNode);
+ }
+ } else {
+ invert = normalizeCondition(rexBuilder, rexNode, scd2Refs);
+ }
+ if (invert == null) {
+ invert = rexNode;
+ }
+ newNodes.add(invert);
+ }
+ if (!newNodes.equals(oriNodes)) {
+ return rexBuilder.makeCall(cnfCondition.getType(), ((RexCall) cnfCondition).getOperator(), newNodes);
+ }
+ return cnfCondition;
}
- private boolean isNotDistinctFrom(RelNode left, RelNode right, RexNode condition) {
+ private boolean hasEqualJoinPart(RelNode left, RelNode right, RexNode condition) {
final List<Integer> leftKeys = new ArrayList<>();
final List<Integer> rightKeys = new ArrayList<>();
final List<Boolean> filterNulls = new ArrayList<>();
RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys, filterNulls);
-
- for (int i = 0; i < leftKeys.size(); i++) {
- if (!filterNulls.get(i))
- return true;
- }
- return false;
+ return !leftKeys.isEmpty() && !rightKeys.isEmpty();
}
private Join transformJoinCondition(LogicalJoin join, JoinInfo info, RelTraitSet traitSet, RelNode left,
RelNode right) {
List<RexInputRef> refs = isPowerBiInnerJoin(info);
- if (refs == null) {
+ if (refs.isEmpty()) {
return join;
}
@@ -148,8 +196,8 @@ public class KapJoinRule extends ConverterRule {
RelOptCluster cluster = join.getCluster();
int index1 = refs.get(0).getIndex();
int index2 = refs.get(1).getIndex();
- int leftIndex = index1 > index2 ? index2 : index1;
- int rightIndex = index1 > index2 ? index1 : index2;
+ int leftIndex = Math.min(index1, index2);
+ int rightIndex = Math.max(index1, index2);
rightIndex -= left.getRowType().getFieldCount();
JoinInfo newInfo = JoinInfo.of(ImmutableIntList.of(leftIndex), ImmutableIntList.of(rightIndex));
@@ -158,7 +206,7 @@ public class KapJoinRule extends ConverterRule {
newInfo.getEquiCondition(left, right, cluster.getRexBuilder()), newInfo.leftKeys, newInfo.rightKeys,
join.getVariablesSet(), join.getJoinType());
} catch (InvalidRelException e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
}
@@ -180,22 +228,22 @@ public class KapJoinRule extends ConverterRule {
*/
private List<RexInputRef> isPowerBiInnerJoin(JoinInfo info) {
if (info.isEqui()) {
- return null;
+ return Collections.emptyList();
}
// 1. top call is OR
RexNode root = info.getRemaining(null);
if (!(root instanceof RexCall && root.getKind() == SqlKind.OR)) {
- return null;
+ return Collections.emptyList();
}
// 2. operands are ANDs
RexCall rootCall = (RexCall) root;
if (rootCall.operands.size() != 2) {
- return null;
+ return Collections.emptyList();
}
if (!(isOperandSqlAnd(rootCall, 0) && isOperandSqlAnd(rootCall, 1))) {
- return null;
+ return Collections.emptyList();
}
// 3. which operand contains two IS_NULL checks
@@ -209,25 +257,25 @@ public class KapJoinRule extends ConverterRule {
twoNullCall = rightCall;
notNullCall = leftCall;
} else {
- return null;
+ return Collections.emptyList();
}
// 4. two column refs
RexCall isNull1 = (RexCall) twoNullCall.operands.get(0);
RexCall isNull2 = (RexCall) twoNullCall.operands.get(1);
if (!(isOperandInputRef(isNull1, 0) && isOperandInputRef(isNull2, 0))) {
- return null;
+ return Collections.emptyList();
}
Set<RexInputRef> refs = Sets.newHashSet((RexInputRef) isNull1.operands.get(0),
(RexInputRef) isNull2.operands.get(0));
if (refs.size() != 2) {
- return null;
+ return Collections.emptyList();
}
// 5. equal not null
if (notNullCall.operands.size() != 3) {
- return null;
+ return Collections.emptyList();
}
RexCall equalCall, notNull1, notNull2;
@@ -247,23 +295,23 @@ public class KapJoinRule extends ConverterRule {
notNull1 = (RexCall) notNullCall.operands.get(0);
notNull2 = (RexCall) notNullCall.operands.get(1);
} else {
- return null;
+ return Collections.emptyList();
}
if (equalCall.operands.get(0).equals(equalCall.operands.get(1))) {
- return null;
+ return Collections.emptyList();
}
if (!(refs.contains(equalCall.operands.get(0)) && refs.contains(equalCall.operands.get(1)))) {
- return null;
+ return Collections.emptyList();
}
if (notNull1.operands.get(0).equals(notNull2.operands.get(0))) {
- return null;
+ return Collections.emptyList();
}
if (!(refs.contains(notNull1.operands.get(0)) && refs.contains(notNull2.operands.get(0)))) {
- return null;
+ return Collections.emptyList();
}
return Lists.newArrayList(refs);
@@ -296,4 +344,70 @@ public class KapJoinRule extends ConverterRule {
private boolean isOperandRexCall(RexCall call, int ordinal) {
return call.operands.get(ordinal) instanceof RexCall;
}
+
+ private boolean isScd2JoinCondition(JoinInfo joinInfo, LogicalJoin join, List<RexInputRef> scd2Refs) {
+ if (joinInfo.isEqui() || !isScd2Enabled) {
+ return false;
+ }
+ RexBuilder rexBuilder = join.getCluster().getRexBuilder();
+ RexNode remaining = joinInfo.getRemaining(rexBuilder);
+
+ if (remaining == null) {
+ return false;
+ }
+ RexNode cnf = RexUtil.toCnf(rexBuilder, joinInfo.getRemaining(rexBuilder));
+ if (!(cnf instanceof RexCall) || cnf.getKind() != SqlKind.AND) {
+ return false;
+ }
+
+ List<RexNode> nodeList = ((RexCall) cnf).getOperands();
+ if (nodeList.size() % 2 != 0) {
+ return false;
+ }
+ Map<RexInputRef, List<RexCall>> refOpMap = Maps.newHashMap();
+ for (RexNode rexNode : nodeList) {
+ if (!rexNode.isA(SCD2_KINDS)) {
+ return false;
+ }
+ RexCall call = (RexCall) rexNode;
+ RexInputRef left = extractInputRef(call.getOperands().get(0));
+ RexInputRef right = extractInputRef(call.getOperands().get(1));
+ if (left == null || right == null) {
+ return false;
+ }
+ refOpMap.putIfAbsent(left, Lists.newArrayList());
+ refOpMap.putIfAbsent(right, Lists.newArrayList());
+ refOpMap.get(left).add((RexCall) rexBuilder.makeCall(call.getOperator(), left, right));
+ refOpMap.get(right).add((RexCall) rexBuilder.makeCall(inverse(call.getOperator()), right, left));
+ }
+ refOpMap.entrySet().removeIf(entry -> entry.getValue().size() != 2);
+ AtomicInteger allSize = new AtomicInteger();
+ refOpMap.forEach((k, pairs) -> {
+ SqlOperator first = pairs.get(0).getOperator();
+ SqlOperator second = pairs.get(1).getOperator();
+ if (!first.equals(second) && ALLOWED_PAIRS.contains(ImmutableSet.of(first, second))) {
+ allSize.getAndAdd(2);
+ }
+ });
+ boolean isScd2 = allSize.get() == nodeList.size();
+ if (isScd2) {
+ scd2Refs.addAll(refOpMap.keySet());
+ }
+ return isScd2;
+ }
+
+ private RexInputRef extractInputRef(RexNode node) {
+ if (node.isA(SqlKind.INPUT_REF)) {
+ return (RexInputRef) node;
+ }
+ if (node.isA(SqlKind.CAST) && (node instanceof RexCall)) {
+ RexCall call = (RexCall) node;
+ if (call.operands.size() == 1) {
+ RexNode rexNode = call.operands.get(0);
+ return rexNode.isA(SqlKind.INPUT_REF) ? (RexInputRef) rexNode : null;
+ }
+
+ }
+ return null;
+ }
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java b/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
index bac1fbf81b..5855f1d1f4 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
@@ -252,8 +252,8 @@ public class PlannerFactory {
}
private ConverterRule selectJoinRuleByConfig() {
- return kylinConfig.isQueryNonEquiJoinModelEnabled() && (!BackdoorToggles.getIsQueryFromAutoModeling()
- || BackdoorToggles.getIsQueryNonEquiJoinModelEnabled() || KylinConfig.getInstanceFromEnv().isUTEnv())
+ return (kylinConfig.isQueryNonEquiJoinModelEnabled() && !BackdoorToggles.getIsQueryFromAutoModeling())
+ || (kylinConfig.isNonEquiJoinRecommendationEnabled() && BackdoorToggles.getIsQueryFromAutoModeling()) //
? KapJoinRule.NON_EQUI_INSTANCE
: KapJoinRule.INSTANCE;
}
@@ -282,12 +282,8 @@ public class PlannerFactory {
String field = split[1];
try {
func.apply((RelOptRule) Class.forName(clazz).getDeclaredField(field).get(null));
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
- } catch (NoSuchFieldException e) {
- throw new RuntimeException(e);
+ } catch (IllegalAccessException | ClassNotFoundException | NoSuchFieldException e) {
+ throw new IllegalStateException(e);
}
}
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index 442eea7faf..18f912ea48 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -107,8 +107,10 @@ public class QueryExec {
private final SimpleDataContext dataContext;
private final boolean allowAlternativeQueryPlan;
private final CalciteSchema rootSchema;
+ private final String project;
public QueryExec(String project, KylinConfig kylinConfig, boolean allowAlternativeQueryPlan) {
+ this.project = project;
this.kylinConfig = kylinConfig;
config = KECalciteConfig.fromKapConfig(kylinConfig);
schemaFactory = new ProjectSchemaFactory(project, kylinConfig);
@@ -172,6 +174,7 @@ public class QueryExec {
public QueryResult executeQuery(String sql) throws SQLException {
magicDirts(sql);
QueryContext queryContext = QueryContext.current();
+ queryContext.setProject(project);
try {
if (kylinConfig.isQueryDryRunEnabled()) {
logger.trace("Dry run Mode.");
@@ -421,8 +424,7 @@ public class QueryExec {
}
private ExecuteResult tryEnhancedAggPushDown(RelNode rel) {
- String project = QueryContext.current().getProject();
- if (project != null && NProjectManager.getProjectConfig(project).isEnhancedAggPushDownEnabled()) {
+ if (NProjectManager.getProjectConfig(project).isEnhancedAggPushDownEnabled()) {
Collection<RelOptRule> postOptRules = new LinkedHashSet<>();
postOptRules.addAll(HepUtils.AggPushDownRules);
return sparderQueryOptimized(rel, 1, postOptRules);
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
index ea4624cecb..988eca606a 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
@@ -22,6 +22,7 @@ import java.sql.SQLException;
import java.util.List;
import org.apache.calcite.DataContext;
+import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.ForceToTieredStorage;
@@ -36,6 +37,7 @@ import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.IndexEntity;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.engine.exec.ExecuteResult;
import org.apache.kylin.query.engine.exec.QueryPlanExec;
import org.apache.kylin.query.engine.exec.calcite.CalciteQueryPlanExec;
@@ -72,8 +74,15 @@ public class SparderQueryPlanExec implements QueryPlanExec {
QueryContext.current().record("end_plan");
QueryContext.current().getQueryTagInfo().setWithoutSyntaxError(true);
- QueryContextCutter.selectRealization(rel, BackdoorToggles.getIsQueryFromAutoModeling());
- ContextUtil.dumpCalcitePlan("EXECUTION PLAN AFTER (SparderQueryPlanExec) SELECT REALIZATION IS SET", rel, log);
+ QueryContextCutter.selectRealization(QueryContext.current().getProject(), rel,
+ BackdoorToggles.getIsQueryFromAutoModeling());
+ String msg = "EXECUTION PLAN AFTER (SparderQueryPlanExec) SELECT REALIZATION IS SET";
+ ContextUtil.dumpCalcitePlan(msg, rel, log);
+
+ // used for printing query plan when diagnosing query problem
+ if (NProjectManager.getProjectConfig(QueryContext.current().getProject()).isPrintQueryPlanEnabled()) {
+ log.info(RelOptUtil.toString(rel));
+ }
val contexts = ContextUtil.listContexts();
for (OLAPContext context : contexts) {
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
index ae748fa863..e94f924c46 100644
--- a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
+++ b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
@@ -23,6 +23,7 @@ import static org.apache.kylin.common.exception.ServerErrorCode.STREAMING_TABLE_
import java.util.List;
import java.util.Locale;
+import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinRuntimeException;
@@ -30,6 +31,7 @@ import org.apache.kylin.common.exception.KylinTimeoutException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.exception.UserStopQueryException;
@@ -68,13 +70,17 @@ public class QueryContextCutter {
* to get multiple smaller OlapContexts and use the previous steps to continue matching.
* @return Each of the returned OlapContexts matches an index, or throws an exception.
*/
- public static List<OLAPContext> selectRealization(RelNode root, boolean isReCutBanned) {
+ public static List<OLAPContext> selectRealization(String project, RelNode root, boolean isReCutBanned) {
ContextInitialCutStrategy firstRoundStrategy = new ContextInitialCutStrategy();
ContextReCutStrategy reCutStrategy = new ContextReCutStrategy();
QueryContextCutter.cutContext(firstRoundStrategy, (KapRel) root.getInput(0), root);
int retryCutTimes = 0;
+ boolean printPlan = NProjectManager.getProjectConfig(project).isPrintQueryPlanEnabled();
while (retryCutTimes++ < MAX_RETRY_TIMES_OF_CONTEXT_CUT) {
+ if (printPlan) {
+ log.info("Cut context {} time(s)\n", RelOptUtil.toString(root));
+ }
try {
fillOlapContextPropertiesWithRelTree(root);
List<OLAPContext> olapContexts = chooseCandidate();
@@ -83,11 +89,7 @@ public class QueryContextCutter {
}
return olapContexts;
} catch (NoRealizationFoundException | NoStreamingRealizationFoundException e) {
- if (isReCutBanned && e instanceof NoStreamingRealizationFoundException) {
- checkStreamingTableWithAutoModeling();
- } else if (isReCutBanned || e instanceof NoStreamingRealizationFoundException || QueryContext.current().isDryRun()) {
- throw e;
- }
+ throwIfReCutBanned(isReCutBanned, e);
reCutStrategy.tryCutToSmallerContexts(root, e);
} catch (UserStopQueryException | KylinTimeoutException | KylinRuntimeException e) {
throw e;
@@ -109,6 +111,14 @@ public class QueryContextCutter {
throw new NoRealizationFoundException(errorMsg);
}
+ private static void throwIfReCutBanned(boolean isReCutBanned, RuntimeException e) {
+ if (isReCutBanned && e instanceof NoStreamingRealizationFoundException) {
+ checkStreamingTableWithAutoModeling();
+ } else if (isReCutBanned || e instanceof NoStreamingRealizationFoundException) {
+ throw e;
+ }
+ }
+
private static void fillOlapContextPropertiesWithRelTree(RelNode queryRoot) {
// post-order travel children
OLAPRel.OLAPImplementor kapImplementor = new OLAPRel.OLAPImplementor();
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
index 4d535bbea0..dd50f2388f 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
@@ -38,6 +38,7 @@ import org.apache.kylin.common.exception.NewQueryRefuseException;
import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.exception.TargetSegmentNotFoundException;
+import org.apache.kylin.common.exception.code.ErrorCodeServer;
import org.apache.kylin.common.persistence.InMemResourceStore;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.TransactionException;
@@ -120,7 +121,7 @@ public class QueryRoutingEngineTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(2, pushdownCount);
// Throw Exception When push down
- Mockito.doThrow(new KylinException(QueryErrorCode.SCD2_DUPLICATE_JOIN_COL, "")).when(queryRoutingEngine)
+ Mockito.doThrow(new KylinException(ErrorCodeServer.DUPLICATE_MODEL_JOIN_CONDITIONS)).when(queryRoutingEngine)
.tryPushDownSelectQuery(Mockito.any(), Mockito.any(), Mockito.anyBoolean());
try {
queryRoutingEngine.queryWithSqlMassage(queryParams);
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java
index 453ae67d60..e003038f2a 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/SelectRealizationTest.java
@@ -65,7 +65,8 @@ class SelectRealizationTest {
void testDerivedFromSameContext() throws SqlParseException {
val kylinConfig = getTestConfig();
val config = KECalciteConfig.fromKapConfig(kylinConfig);
- val schemaFactory = new ProjectSchemaFactory("default", kylinConfig);
+ String prj = "default";
+ val schemaFactory = new ProjectSchemaFactory(prj, kylinConfig);
val rootSchema = schemaFactory.createProjectRootSchema();
String defaultSchemaName = schemaFactory.getDefaultSchema();
val catalogReader = createCatalogReader(config, rootSchema, defaultSchemaName);
@@ -81,7 +82,8 @@ class SelectRealizationTest {
+ "FROM \"SSB\".\"LINEORDER\" \"LINEORDER\"\n" + "GROUP BY 1.1000000000000001 ) \"t0\"\n"
+ "ON LINEORDER.LO_ORDERDATE = t0.X_measure__0\n" + "GROUP BY 1.1000000000000001");
RelNode node = queryOptimizer.optimize(relRoot).rel;
- val olapContexts = QueryContextCutter.selectRealization(node, BackdoorToggles.getIsQueryFromAutoModeling());
+ val olapContexts = QueryContextCutter.selectRealization(prj, node,
+ BackdoorToggles.getIsQueryFromAutoModeling());
Assertions.assertNotNull(olapContexts);
Assertions.assertFalse(olapContexts.isEmpty());
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
index fac5914bf6..44580d3a34 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
@@ -23,6 +23,7 @@ import java.sql.Timestamp
import java.util
import java.util.concurrent.{Callable, Executors, TimeUnit, TimeoutException}
import java.util.{UUID, List => JList}
+
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.common.util.{DateFormat, HadoopUtil, Pair}
import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
@@ -80,7 +81,11 @@ object SparkSqlClient {
null
}
ss.sessionState.conf.setLocalProperty(DEFAULT_DB, db)
- val df = QueryResultMasks.maskResult(ss.sql(sqlToRun))
+ val dfOfPushDown = ss.sql(sqlToRun)
+ if (NProjectManager.getProjectConfig(project).isPrintQueryPlanEnabled) {
+ logger.info(dfOfPushDown.queryExecution.logical.toString())
+ }
+ val df = QueryResultMasks.maskResult(dfOfPushDown)
logger.info("SparkSQL returned result DataFrame")
QueryContext.current().record("to_spark_plan")
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
index 1cde2767b9..37f527e20a 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
@@ -83,7 +83,11 @@ object TableScanPlan extends LogEx {
// The reason why we use Project to package it here is because the output method of Union needs to be analyzed,
// so it needs to be packaged by Project so that subsequent operators can easily obtain the output information of nodes.
- Project(plans.head.output, Union(plans))
+ if (plans.size == 1) {
+ plans.head
+ } else {
+ Project(plans.head.output, Union(plans))
+ }
}
def createMetadataTable(rel: KapRel): LogicalPlan = {
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/util/SparderDerivedUtil.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/util/SparderDerivedUtil.scala
index a8b38dccb3..ed99306148 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/util/SparderDerivedUtil.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/util/SparderDerivedUtil.scala
@@ -25,8 +25,8 @@ import org.apache.kylin.guava30.shaded.common.collect.Maps
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.model.DeriveInfo.DeriveType
-import org.apache.kylin.metadata.model.NonEquiJoinCondition.SimplifiedNonEquiJoinCondition
-import org.apache.kylin.metadata.model.util.scd2.SCD2NonEquiCondSimplification
+import org.apache.kylin.metadata.model.NonEquiJoinCondition.SimplifiedJoinCondition
+import org.apache.kylin.metadata.model.util.scd2.Scd2Simplifier
import org.apache.kylin.metadata.model.{DeriveInfo, NDataModel, NTableMetadataManager, TblColRef}
import org.apache.kylin.metadata.tuple.TupleInfo
import org.apache.spark.sql.catalyst.plans.JoinType
@@ -220,8 +220,8 @@ case class SparderDerivedUtil(gtInfoTableName: String,
}
if (derivedInfo.deriveType == DeriveType.LOOKUP_NON_EQUI) {
- val simplifiedCond = SCD2NonEquiCondSimplification.INSTANCE
- .convertToSimplifiedSCD2Cond(derivedInfo.join)
+ val simplifiedCond = Scd2Simplifier.INSTANCE
+ .simplifyScd2Conditions(derivedInfo.join)
.getSimplifiedNonEquiJoinConditions
joinCol = genNonEquiJoinColumn(newNameLookupPlan, gTInfoNames, joinCol, simplifiedCond.asScala)
}
@@ -231,7 +231,7 @@ case class SparderDerivedUtil(gtInfoTableName: String,
def genNonEquiJoinColumn(newNameLookupPlan: LogicalPlan,
gTInfoNames: Array[String],
colOrigin: Column,
- simplifiedConds: mutable.Buffer[SimplifiedNonEquiJoinCondition]): Column = {
+ simplifiedConds: mutable.Buffer[SimplifiedJoinCondition]): Column = {
var joinCol = colOrigin
for (simplifiedCond <- simplifiedConds) {
@@ -240,8 +240,10 @@ case class SparderDerivedUtil(gtInfoTableName: String,
val colOp = simplifiedCond.getOp
val newCol = colOp match {
+ case SqlKind.LESS_THAN_OR_EQUAL => colFk.<=(colPk)
case SqlKind.GREATER_THAN_OR_EQUAL => colFk.>=(colPk)
case SqlKind.LESS_THAN => colFk.<(colPk)
+ case SqlKind.GREATER_THAN => colFk.>(colPk)
}
joinCol = joinCol.and(newCol)
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/MaintainModeTool.java b/src/tool/src/main/java/org/apache/kylin/tool/MaintainModeTool.java
index 29ac7bfe78..ebcf193f59 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/MaintainModeTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/MaintainModeTool.java
@@ -24,6 +24,8 @@ import java.util.stream.Collectors;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
@@ -37,8 +39,6 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.tool.constant.StringConstant;
import org.apache.kylin.tool.util.ToolMainWrapper;
-import org.springframework.util.CollectionUtils;
-import org.springframework.util.StringUtils;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/routine/MaintainModeToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/routine/MaintainModeToolTest.java
index 4c21dc9021..90fae47b51 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/routine/MaintainModeToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/routine/MaintainModeToolTest.java
@@ -24,30 +24,42 @@ import org.apache.kylin.junit.annotation.JdbcMetadataInfo;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.tool.MaintainModeTool;
-import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
@MetadataInfo(onlyProps = true)
@JdbcMetadataInfo
-public class MaintainModeToolTest {
+class MaintainModeToolTest {
@Test
- public void testForceToExit() {
- MaintainModeTool maintainModeTool = new MaintainModeTool();
- maintainModeTool.execute(new String[] { "-off", "--force" });
+ void testForceToExit() {
+ try {
+ MaintainModeTool maintainModeTool = new MaintainModeTool();
+ maintainModeTool.execute(new String[] { "-off", "--force" });
+ } catch (Exception e) {
+ log.info("Something wrong when running testForceToExit\n", e);
+ throw new RuntimeException(e);
+ }
}
@Test
- public void testEnterMaintainMode() {
- MaintainModeTool maintainModeTool = new MaintainModeTool();
- maintainModeTool.execute(new String[] { "-on", "-reason", "test" });
+ void testEnterMaintainMode() {
+ try {
+ MaintainModeTool maintainModeTool = new MaintainModeTool();
+ maintainModeTool.execute(new String[] { "-on", "-reason", "test" });
+ } catch (Exception e) {
+ log.info("Something wrong when running testEnterMaintainMode\n", e);
+ throw new RuntimeException(e);
+ }
}
@Test
- public void testEnterMaintainModeEpochCheck() {
+ void testEnterMaintainModeEpochCheck() {
EpochManager epochManager = EpochManager.getInstance();
epochManager.tryUpdateEpoch(EpochManager.GLOBAL, true);
@@ -57,22 +69,22 @@ public class MaintainModeToolTest {
ReflectionTestUtils.invokeMethod(epochManager, "insertOrUpdateEpoch", globalEpoch);
- Assert.assertEquals(epochManager.getGlobalEpoch().getEpochId(), id);
+ Assertions.assertEquals(id, epochManager.getGlobalEpoch().getEpochId());
MaintainModeTool maintainModeTool = new MaintainModeTool();
maintainModeTool.execute(new String[] { "-on", "-reason", "test" });
- Assert.assertEquals(epochManager.getGlobalEpoch().getEpochId(), id + 1);
+ Assertions.assertEquals(id + 1, epochManager.getGlobalEpoch().getEpochId());
maintainModeTool.execute(new String[] { "-off", "--force" });
- Assert.assertEquals(epochManager.getGlobalEpoch().getEpochId(), id + 2);
+ Assertions.assertEquals(id + 2, epochManager.getGlobalEpoch().getEpochId());
}
@Test
- public void testCleanProject() {
+ void testCleanProject() {
MaintainModeTool maintainModeTool = new MaintainModeTool();
maintainModeTool.execute(new String[] { "-on", "-reason", "test", "-p", "notExistP1" });
- Assert.assertEquals(getEpochStore().list().size(), 2);
+ Assertions.assertEquals(2, getEpochStore().list().size());
}
EpochStore getEpochStore() {
(kylin) 25/35: KYLIN-5832 Support queryDetect api
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d7f37675e2de44478da78d5ae7b4b70fd043b13f
Author: haocheni <ha...@qq.com>
AuthorDate: Fri Sep 22 19:21:38 2023 +0800
KYLIN-5832 Support queryDetect api
---
.../java/org/apache/kylin/common/QueryContext.java | 1 +
.../kylin/rest/controller/NQueryController.java | 9 +
.../kylin/rest/request/QueryDetectRequest.java | 33 +++
.../kylin/rest/response/QueryDetectResponse.java | 156 ++++++++++++
.../apache/kylin/rest/service/QueryService.java | 37 ++-
.../kylin/rest/service/QueryServiceTest.java | 278 ++++++++++++++++++++-
.../kylin/query/engine/QueryRoutingEngine.java | 7 +
.../engine/exec/sparder/SparderQueryPlanExec.java | 6 +
.../kylin/query/engine/QueryRoutingEngineTest.java | 34 +++
9 files changed, 557 insertions(+), 4 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 592df87f68..b112591e27 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -389,6 +389,7 @@ public class QueryContext implements Closeable {
private boolean isRefused;
private boolean includeHeader;
private boolean isVacant;
+ private boolean isQueryDetect;
}
@Getter
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index 6d69ea475f..1f56073647 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -73,12 +73,14 @@ import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLFormatRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.request.SyncFileSegmentsRequest;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.QueryHistoryFiltersResponse;
import org.apache.kylin.rest.response.QueryStatisticsResponse;
import org.apache.kylin.rest.response.SQLResponse;
@@ -662,6 +664,13 @@ public class NQueryController extends NBasicController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, queryService.format(request.getSqls()), "");
}
+ @ApiOperation(value = "queryDetect", tags = { "QE" })
+ @PostMapping("/detection")
+ public EnvelopeResponse<QueryDetectResponse> queryDetect(@RequestBody QueryDetectRequest request) {
+ checkProjectName(request.getProject());
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, queryService.queryDetect(request), "");
+ }
+
private void checkQueryName(String queryName) {
if (!queryNamePattern.matcher(queryName).matches()) {
throw new KylinException(INVALID_NAME, MsgPicker.getMsg().getInvalidQueryName());
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java b/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
new file mode 100644
index 0000000000..9ee03f4b3f
--- /dev/null
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class QueryDetectRequest {
+ private String sql;
+ private String project;
+ private Integer offset = 0;
+ private Integer limit = 500;
+}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java b/src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java
new file mode 100644
index 0000000000..b6548101d9
--- /dev/null
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.response;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
+import org.apache.kylin.metadata.cube.model.LayoutEntity;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
+import org.apache.kylin.metadata.query.NativeQueryRealization;
+import org.apache.kylin.metadata.query.QueryHistory;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonAutoDetect(fieldVisibility = NONE, getterVisibility = NONE, isGetterVisibility = NONE, setterVisibility = NONE)
+public class QueryDetectResponse {
+ @JsonProperty("is_exception")
+ private boolean isException = false;
+
+ @JsonProperty("exception_message")
+ private String exceptionMessage;
+
+ @JsonProperty("query_id")
+ private String queryId;
+
+ @JsonProperty("is_push_down")
+ private boolean isPushDown = false;
+
+ @JsonProperty("is_post_aggregation")
+ private boolean isPostAggregation = false;
+
+ @JsonProperty("is_table_index")
+ private boolean isTableIndex = false;
+
+ @JsonProperty("is_base_index")
+ private boolean isBaseIndex = false;
+
+ @JsonProperty("is_cache")
+ private boolean isCache = false;
+
+ @JsonProperty("is_constants")
+ private boolean isConstants = false;
+
+ @JsonProperty("realizations")
+ private List<IndexInfo> realizations = Lists.newArrayList();
+
+ public QueryDetectResponse buildExceptionResponse(SQLResponse sqlResponse) {
+ this.isException = true;
+ this.exceptionMessage = sqlResponse.getExceptionMessage();
+ this.queryId = sqlResponse.getQueryId();
+ this.isCache = sqlResponse.isHitExceptionCache();
+ this.realizations = Lists.newArrayList();
+ return this;
+ }
+
+ public QueryDetectResponse buildResponse(String project, SQLResponse sqlResponse, QueryContext queryContext) {
+ // build RealizationVO
+ List<QueryDetectResponse.IndexInfo> indexInfoList = sqlResponse.getNativeRealizations().stream()
+ .map(realization -> new QueryDetectResponse.IndexInfo().buildResponse(realization, project))
+ .collect(Collectors.toList());
+ boolean isConstantQuery = QueryHistory.EngineType.CONSTANTS.name().equals(sqlResponse.getEngineType());
+
+ this.isException = sqlResponse.isException;
+ this.exceptionMessage = sqlResponse.getExceptionMessage();
+ this.queryId = sqlResponse.getQueryId();
+ this.isPushDown = sqlResponse.isQueryPushDown();
+ this.isPostAggregation = !sqlResponse.isQueryPushDown() && !isConstantQuery
+ && !queryContext.getMetrics().isExactlyMatch();
+ // any realization is TableIndex、baseIndex
+ this.isTableIndex = indexInfoList.stream().anyMatch(QueryDetectResponse.IndexInfo::isTableIndex);
+ this.isBaseIndex = indexInfoList.stream().anyMatch(QueryDetectResponse.IndexInfo::isBaseIndex);
+ this.isCache = sqlResponse.isStorageCacheUsed();
+ this.isConstants = isConstantQuery;
+ this.realizations = indexInfoList;
+ return this;
+ }
+
+ @Data
+ @AllArgsConstructor
+ @NoArgsConstructor
+ @JsonAutoDetect(fieldVisibility = NONE, getterVisibility = NONE, isGetterVisibility = NONE, setterVisibility = NONE)
+ public static class IndexInfo {
+ @JsonProperty("model_id")
+ private String modelId;
+
+ @JsonProperty("model_alias")
+ private String modelAlias;
+
+ @JsonProperty("layout_id")
+ private long layoutId;
+
+ @JsonProperty("index_type")
+ private String indexType;
+
+ @JsonProperty("partial_match_model")
+ private boolean partialMatchModel = false;
+
+ @JsonProperty("valid")
+ private boolean valid = true;
+
+ @JsonProperty("is_table_index")
+ private boolean isTableIndex = false;
+
+ @JsonProperty("is_base_index")
+ private boolean isBaseIndex = false;
+
+ public IndexInfo buildResponse(NativeQueryRealization realization, String project) {
+ // calculate isBaseIndex
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ IndexPlan indexPlan = NIndexPlanManager.getInstance(kylinConfig, project)
+ .getIndexPlan(realization.getModelId());
+ LayoutEntity layoutEntity = indexPlan == null ? null : indexPlan.getLayoutEntity(realization.getLayoutId());
+
+ this.modelId = realization.getModelId();
+ this.modelAlias = realization.getModelAlias();
+ this.layoutId = realization.getLayoutId();
+ this.indexType = realization.getIndexType();
+ this.partialMatchModel = realization.isPartialMatchModel();
+ this.valid = realization.isValid();
+ this.isTableIndex = IndexEntity.isTableIndex(realization.getLayoutId());
+ this.isBaseIndex = layoutEntity != null && layoutEntity.isBaseIndex();
+ return this;
+ }
+ }
+}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7b10e1eb67..04bd71ff81 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -136,6 +136,7 @@ import org.apache.kylin.query.exception.UserStopQueryException;
import org.apache.kylin.query.relnode.ContextUtil;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.EscapeDialect;
+import org.apache.kylin.query.util.PrepareSQLUtils;
import org.apache.kylin.query.util.QueryLimiter;
import org.apache.kylin.query.util.QueryModelPriorities;
import org.apache.kylin.query.util.QueryParams;
@@ -150,7 +151,9 @@ import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.config.AppConfig;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.response.SQLResponseTrace;
import org.apache.kylin.rest.response.TableMetaCacheResult;
@@ -158,7 +161,6 @@ import org.apache.kylin.rest.response.TableMetaCacheResultV2;
import org.apache.kylin.rest.security.MutableAclRecord;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
-import org.apache.kylin.query.util.PrepareSQLUtils;
import org.apache.kylin.rest.util.QueryCacheSignatureUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
import org.apache.kylin.rest.util.QueryUtils;
@@ -645,7 +647,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
sqlResponse = QueryUtils.handleTempStatement(sqlRequest, kylinConfig);
// search cache
- if (sqlResponse == null && kylinConfig.isQueryCacheEnabled() && !sqlRequest.isForcedToPushDown()) {
+ if (sqlResponse == null && kylinConfig.isQueryCacheEnabled() && !sqlRequest.isForcedToPushDown()
+ && !queryContext.getQueryTagInfo().isAsyncQuery()) {
sqlResponse = searchCache(sqlRequest, kylinConfig);
}
@@ -754,6 +757,10 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
private void addToQueryHistory(SQLRequest sqlRequest, SQLResponse sqlResponse, String originalSql) {
+ if (QueryContext.current().getQueryTagInfo().isQueryDetect()) {
+ return;
+ }
+
if (!(QueryContext.current().getQueryTagInfo().isAsyncQuery()
&& NProjectManager.getProjectConfig(sqlRequest.getProject()).isUniqueAsyncQueryYarnQueue())) {
try {
@@ -936,8 +943,10 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
private boolean isQueryCacheEnabled(KylinConfig kylinConfig) {
+ boolean isNotQueryDetect = !QueryContext.current().getQueryTagInfo().isQueryDetect();
return checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache disabled in KylinConfig") //
- && checkCondition(!BackdoorToggles.getDisableCache(), "query cache disabled in BackdoorToggles");
+ && checkCondition(!BackdoorToggles.getDisableCache(), "query cache disabled in BackdoorToggles")
+ && checkCondition(isNotQueryDetect, "query cache disabled because it is query detect");
}
private boolean isQueryExceptionCacheEnabled(KylinConfig kylinConfig) {
@@ -1168,6 +1177,28 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}).sorted(Comparator.comparingInt(Pair::getFirst)).map(Pair::getSecond).collect(Collectors.toList());
}
+ public QueryDetectResponse queryDetect(QueryDetectRequest queryDetectRequest) {
+ try (QueryContext queryContext = QueryContext.current()) {
+ // call query method to collect realizations, etc
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ String project = queryDetectRequest.getProject();
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ sqlRequest.setProject(project);
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ SQLResponse sqlResponse = queryWithCache(sqlRequest);
+
+ // build exception response
+ if (sqlResponse.isException()) {
+ return new QueryDetectResponse().buildExceptionResponse(sqlResponse);
+ }
+
+ // build normal response
+ return new QueryDetectResponse().buildResponse(project, sqlResponse, queryContext);
+ }
+ }
+
@SuppressWarnings("checkstyle:methodlength")
private List<TableMetaWithType> doGetMetadataV2(String project, String targetModelName) {
aclEvaluate.checkProjectQueryPermission(project);
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 89fead0d99..b9c5ab91e2 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -82,11 +82,13 @@ import org.apache.kylin.metadata.acl.AclTCR;
import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.IndexEntity;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
@@ -99,6 +101,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryHistory;
+import org.apache.kylin.metadata.query.QueryMetrics;
import org.apache.kylin.metadata.query.QueryMetricsContext;
import org.apache.kylin.metadata.querymeta.ColumnMeta;
import org.apache.kylin.metadata.querymeta.ColumnMetaWithType;
@@ -127,7 +130,9 @@ import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.security.AclEntityFactory;
import org.apache.kylin.rest.security.AclEntityType;
@@ -169,7 +174,8 @@ import lombok.val;
* @author xduo
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ SpringContext.class, UserGroupInformation.class, SparkSession.class, QueryService.class })
+@PrepareForTest({ SpringContext.class, UserGroupInformation.class, SparkSession.class, QueryService.class,
+ NIndexPlanManager.class, QueryContext.class })
@PowerMockIgnore({ "javax.management.*" })
public class QueryServiceTest extends NLocalFileMetadataTestCase {
@@ -2743,4 +2749,274 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
val queryEntry = result.get();
Assert.assertTrue(queryEntry.getPlannerCancelFlag().isCancelRequested());
}
+
+ @Test
+ public void testQueryDetectException() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql", "default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(true);
+ sqlResponse.setExceptionMessage("exceptionMessage");
+ sqlResponse.setQueryId("queryId");
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new QueryDetectResponse().buildExceptionResponse(sqlResponse);
+
+ Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ QueryDetectResponse queryDetectResponse = queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectPushDown() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql", "default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(true);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList();
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ QueryDetectResponse queryDetectResponse = queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectAggIndex() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql", "default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ NativeQueryRealization realization = new NativeQueryRealization();
+ realization.setModelId("modelId1");
+ realization.setLayoutId(1L);
+ realization.setModelAlias("modelAlias1");
+ realization.setIndexType(QueryMetrics.AGG_INDEX);
+ realization.setPartialMatchModel(false);
+ realization.setValid(true);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList(realization);
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // mock NIndexPlanManager and IndexPlan
+ PowerMockito.mockStatic(NIndexPlanManager.class);
+ NIndexPlanManager nIndexPlanManager = Mockito.mock(NIndexPlanManager.class);
+ IndexPlan indexPlan = Mockito.mock(IndexPlan.class);
+ LayoutEntity layoutEntity = new LayoutEntity();
+ layoutEntity.setBase(true);
+
+ // mock method return value
+ Mockito.when(NIndexPlanManager.getInstance(Mockito.any(), Mockito.any())).thenReturn(nIndexPlanManager);
+ Mockito.when(nIndexPlanManager.getIndexPlan(realization.getModelId())).thenReturn(indexPlan);
+ Mockito.when(indexPlan.getLayoutEntity(realization.getLayoutId())).thenReturn(layoutEntity);
+ Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse = queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectWhenIsConstants() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql", "default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList();
+ sqlResponse.setNativeRealizations(nativeRealizations);
+ sqlResponse.setEngineType(QueryHistory.EngineType.CONSTANTS.name());
+
+ // mock method return value
+ Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse = queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectWhenIsSnapshot() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql", "default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ NativeQueryRealization realization = new NativeQueryRealization();
+ realization.setModelId("modelId1");
+ realization.setLayoutId(-1L);
+ realization.setModelAlias("modelAlias1");
+ realization.setIndexType(QueryMetrics.AGG_INDEX);
+ realization.setPartialMatchModel(false);
+ realization.setValid(true);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList(realization);
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // mock NIndexPlanManager and IndexPlan
+ PowerMockito.mockStatic(NIndexPlanManager.class);
+ NIndexPlanManager nIndexPlanManager = Mockito.mock(NIndexPlanManager.class);
+ IndexPlan indexPlan = Mockito.mock(IndexPlan.class);
+
+ // mock method return value
+ Mockito.when(NIndexPlanManager.getInstance(Mockito.any(), Mockito.any())).thenReturn(nIndexPlanManager);
+ Mockito.when(nIndexPlanManager.getIndexPlan(realization.getModelId())).thenReturn(indexPlan);
+ Mockito.when(indexPlan.getLayoutEntity(realization.getLayoutId())).thenReturn(null);
+ Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse = queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectWhenIndexPlanIsNull() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql", "default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ NativeQueryRealization realization = new NativeQueryRealization();
+ realization.setModelId("modelId1");
+ realization.setLayoutId(-1L);
+ realization.setModelAlias("modelAlias1");
+ realization.setIndexType(QueryMetrics.AGG_INDEX);
+ realization.setPartialMatchModel(false);
+ realization.setValid(true);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList(realization);
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // mock NIndexPlanManager and IndexPlan
+ PowerMockito.mockStatic(NIndexPlanManager.class);
+ NIndexPlanManager nIndexPlanManager = Mockito.mock(NIndexPlanManager.class);
+
+ // mock method return value
+ Mockito.when(NIndexPlanManager.getInstance(Mockito.any(), Mockito.any())).thenReturn(nIndexPlanManager);
+ Mockito.when(nIndexPlanManager.getIndexPlan(realization.getModelId())).thenReturn(null);
+ Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse = queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testAddToQueryHistoryIsQueryDetect() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method = QueryService.class.getDeclaredMethod("addToQueryHistory", SQLRequest.class,
+ SQLResponse.class, String.class);
+ method.setAccessible(true);
+
+ // check void method is ok
+ method.invoke(queryService, null, null, null);
+ }
+ }
+
+ @Test
+ public void testIsQueryCacheEnabledIsQueryDetect() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method = QueryService.class.getDeclaredMethod("isQueryCacheEnabled", KylinConfig.class);
+ method.setAccessible(true);
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ boolean isQueryCacheEnabled = (boolean) method.invoke(queryService, kylinConfig);
+
+ Assert.assertFalse(isQueryCacheEnabled);
+ }
+ }
+
+ @Test
+ public void testNotHitCacheWhenIsAsyncQuery() {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setAsyncQuery(true);
+ String project = "default";
+ String sql = "select * from table";
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(project);
+ sqlRequest.setSql(sql);
+
+ queryService.doQueryWithCache(sqlRequest);
+
+ // async query API can not search cache
+ Assert.assertFalse(queryContext.getQueryTagInfo().isStorageCacheUsed());
+ Assert.assertFalse(queryContext.getQueryTagInfo().isHitExceptionCache());
+ Assert.assertTrue(queryContext.getQueryTagInfo().isAsyncQuery());
+ }
+ }
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index 0e3f86b399..f1e5cf9054 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -54,6 +54,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.StructField;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.engine.data.QueryResult;
import org.apache.kylin.query.exception.BusyQueryException;
@@ -260,6 +261,12 @@ public class QueryRoutingEngine {
private QueryResult pushDownQuery(SQLException sqlException, QueryParams queryParams) throws SQLException {
QueryContext.current().getMetrics().setOlapCause(sqlException);
QueryContext.current().getQueryTagInfo().setPushdown(true);
+ if (QueryContext.current().getQueryTagInfo().isQueryDetect()) {
+ if (sqlException.getCause() instanceof NoRealizationFoundException) {
+ return new QueryResult();
+ }
+ throw sqlException;
+ }
PushdownResult result = null;
try {
result = tryPushDownSelectQuery(queryParams, sqlException, BackdoorToggles.getPrepareOnly());
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
index 988eca606a..07c18197eb 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
@@ -76,6 +76,7 @@ public class SparderQueryPlanExec implements QueryPlanExec {
QueryContextCutter.selectRealization(QueryContext.current().getProject(), rel,
BackdoorToggles.getIsQueryFromAutoModeling());
+
String msg = "EXECUTION PLAN AFTER (SparderQueryPlanExec) SELECT REALIZATION IS SET";
ContextUtil.dumpCalcitePlan(msg, rel, log);
@@ -107,6 +108,11 @@ public class SparderQueryPlanExec implements QueryPlanExec {
// rewrite
rewrite(rel);
+ // query detect
+ if (QueryContext.current().getQueryTagInfo().isQueryDetect()) {
+ return new ExecuteResult(Lists.newArrayList(), 0);
+ }
+
// submit rel and dataContext to query engine
return internalCompute(new SparkEngine(), dataContext, rel.getInput(0));
}
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
index dd50f2388f..fa929a34b2 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.query.engine;
import static org.apache.kylin.query.engine.QueryRoutingEngine.SPARK_JOB_FAILED;
import static org.apache.kylin.query.engine.QueryRoutingEngine.SPARK_MEM_LIMIT_EXCEEDED;
+import java.lang.reflect.Method;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Time;
@@ -43,6 +44,7 @@ import org.apache.kylin.common.persistence.InMemResourceStore;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.QueryExtension;
import org.apache.kylin.query.engine.data.QueryResult;
@@ -485,4 +487,36 @@ public class QueryRoutingEngineTest extends NLocalFileMetadataTestCase {
slowQueryDetector.queryEnd();
}
}
+
+ @Test
+ public void testPushDownQueryWhenIsQueryDetect() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method = QueryRoutingEngine.class.getDeclaredMethod("pushDownQuery", SQLException.class,
+ QueryParams.class);
+ method.setAccessible(true);
+
+ try {
+ method.invoke(queryRoutingEngine, new SQLException(), null);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof SQLException);
+ Assert.assertTrue(QueryContext.current().getQueryTagInfo().isPushdown());
+ }
+ }
+ }
+
+ @Test
+ public void testPushDownQueryWhenIsQueryDetectAndPushDown() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method = QueryRoutingEngine.class.getDeclaredMethod("pushDownQuery", SQLException.class,
+ QueryParams.class);
+ method.setAccessible(true);
+
+ Object result = method.invoke(queryRoutingEngine, new SQLException(new NoRealizationFoundException("")),
+ null);
+ Assert.assertNotNull(result);
+ }
+ }
}
(kylin) 23/35: KYLIN-5829 Add a tool for generating sql from dumped metadata
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f9255a515d38edafd099c81712c0b6ff8e7038d7
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Mon Jun 19 11:41:51 2023 +0800
KYLIN-5829 Add a tool for generating sql from dumped metadata
---
build/bin/create-table.sh | 1 -
build/conf/kylin-tools-diag-log4j.xml | 6 ++
.../kylin/tool}/util/CreateTableFromJson.java | 31 +++++-----
.../kylin/tool/util/CreateTableFromJsonTest.java | 68 ++++++++++++++++++++++
4 files changed, 89 insertions(+), 17 deletions(-)
diff --git a/build/bin/create-table.sh b/build/bin/create-table.sh
index 5fbc49541d..a21e39036e 100755
--- a/build/bin/create-table.sh
+++ b/build/bin/create-table.sh
@@ -16,7 +16,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
if [ -z $KYLIN_HOME ];then
export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
fi
diff --git a/build/conf/kylin-tools-diag-log4j.xml b/build/conf/kylin-tools-diag-log4j.xml
index 728e50d2ca..227e71cdd1 100644
--- a/build/conf/kylin-tools-diag-log4j.xml
+++ b/build/conf/kylin-tools-diag-log4j.xml
@@ -28,6 +28,9 @@
<Console name="stdout" target="SYSTEM_OUT">
<PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p %X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
+ <File name="createTableSql" fileName="${env:KYLIN_HOME}/logs/create_table.sql">
+ <PatternLayout pattern="%msg%n"/>
+ </File>
</Appenders>
<Loggers>
<Root level="DEBUG">
@@ -38,5 +41,8 @@
<Logger name="org.springframework.security" level="INFO"/>
<Logger name="io.kyligence" level="DEBUG"/>
<Logger name="org.apache.kylin" level="DEBUG"/>
+ <Logger name="org.apache.kylin.tool.util.CreateTableFromJson" level="INFO">
+ <AppenderRef ref="createTableSql"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java b/src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
similarity index 81%
rename from src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
rename to src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
index 2ee7efef22..67148ad608 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
@@ -16,10 +16,13 @@
* limitations under the License.
*/
-package org.apache.kylin.rest.util;
+package org.apache.kylin.tool.util;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -29,20 +32,16 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* A tool to generate database and tables from metadata backups of customer
*/
+@Slf4j
public class CreateTableFromJson {
private static final String QUOTE = "`";
- private static final Map<String, String> TYPE_MAP = Maps.newHashMap();
-
- private static final Logger logger = LoggerFactory.getLogger(CreateTableFromJson.class);
+ private static final Map<String, String> TYPE_MAP = new HashMap<>();
static {
TYPE_MAP.put("integer", "int");
@@ -54,23 +53,23 @@ public class CreateTableFromJson {
String pathDir = args[0];
Map<String, List<String>> map = createDbAndTables(pathDir);
- map.forEach((k, v) -> {
- logger.info(k);
- v.forEach(logger::info);
+ map.forEach((db, tables) -> {
+ for (String s : db.split("\n")) {
+ log.info(s);
+ }
+ tables.forEach(log::info);
});
-
- logger.info("\n\n\n\n\n");
}
// the path is /{metadata_backup_path}/{project_name}/table/
private static Map<String, List<String>> createDbAndTables(String pathDir) throws IOException {
- Map<String, List<String>> map = Maps.newHashMap();
+ Map<String, List<String>> map = new LinkedHashMap<>();
File file = new File(pathDir).getAbsoluteFile();
File[] files = file.listFiles();
for (File f : Objects.requireNonNull(files)) {
final TableDesc tableDesc = JsonUtil.readValue(f, TableDesc.class);
- List<String> columnNameTypeList = Lists.newArrayList();
+ List<String> columnNameTypeList = new ArrayList<>();
for (ColumnDesc column : tableDesc.getColumns()) {
String name = column.getName();
String type = convert(column.getDatatype());
@@ -79,7 +78,7 @@ public class CreateTableFromJson {
String databaseSql = String.format(Locale.ROOT, "create database %s;%nuse %s;",
quote(tableDesc.getDatabase()), quote(tableDesc.getDatabase()));
- map.putIfAbsent(databaseSql, Lists.newArrayList());
+ map.putIfAbsent(databaseSql, new ArrayList<>());
String tableSql = createTableSql(tableDesc.getName(), columnNameTypeList);
map.get(databaseSql).add(tableSql);
}
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java b/src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java
new file mode 100644
index 0000000000..7d729aee9f
--- /dev/null
+++ b/src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class CreateTableFromJsonTest {
+
+ String path = "src/test/resources/table_cc_cleanup/metadata/AL_4144/table/";
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private PrintStream originPrintStream;
+ private static final String REGEX = "util.CreateTableFromJson :";
+
+ @BeforeEach
+ void setUpStreams() {
+ originPrintStream = System.out;
+ System.setOut(new PrintStream(outContent));
+ }
+
+ @AfterEach
+ void cleanUpStreams() {
+ System.setOut(originPrintStream);
+ }
+
+ @Test
+ void test() throws IOException {
+ CreateTableFromJson.main(new String[] { path });
+ String result = outContent.toString();
+ if (StringUtils.isBlank(result)) {
+ Assertions.assertTrue(true);
+ } else if ("true".equals(System.getProperty("junit.run.local"))) {
+ // with -Djunit.run.local=true, this branch is more convenient for dev
+ String[] splits = result.split("\n");
+ for (String str : splits) {
+ String s = str.split(REGEX)[1];
+ originPrintStream.println(s);
+ }
+ } else {
+ String[] splits = result.split("\n");
+ Assertions.assertTrue(splits[0].contains("create database `CAP`;"));
+ Assertions.assertTrue(splits[1].contains("use `CAP`;"));
+ }
+ }
+}
(kylin) 03/35: KYLIN-5795 Remove the check of `hadoop.tmp.dir`
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 91383f54d5c70bf6ce0f3f641dcd186de193029b
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Mon Aug 28 15:41:05 2023 +0800
KYLIN-5795 Remove the check of `hadoop.tmp.dir`
---
.../src/main/java/org/apache/kylin/common/util/HadoopUtil.java | 4 ----
1 file changed, 4 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 8e4250fc0f..0e3ac41ae5 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -98,10 +98,6 @@ public class HadoopUtil {
}
public static Configuration healSickConfig(Configuration conf) {
- // https://issues.apache.org/jira/browse/KYLIN-953
- if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
- conf.set("hadoop.tmp.dir", "/tmp");
- }
// https://issues.apache.org/jira/browse/KYLIN-3064
conf.set("yarn.timeline-service.enabled", "false");
(kylin) 12/35: KYLIN-5805 Refactor project lock
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 085d7072167bf199094833ef73f0d9cc5741b075
Author: Zhiting Guo <35...@users.noreply.github.com>
AuthorDate: Fri Sep 1 15:59:08 2023 +0800
KYLIN-5805 Refactor project lock
---------
Co-authored-by: junqing.cai <ju...@kyligence.io>
Co-authored-by: Jiale He <ji...@gmail.com>
Co-authored-by: Zhiting Guo <zh...@kyligence.io>
---
build/bin/create-table.sh | 33 +++
build/conf/kylin-tools-log4j.xml | 6 +
.../kap/secondstorage/metadata/Manager.java | 23 +-
.../kap/secondstorage/metadata/NodeGroup.java | 5 +
.../kap/secondstorage/metadata/TablePlan.java | 5 +
.../kylin/rest/controller/NSystemController.java | 33 +++
.../apache/kylin/rest/config/AppInitializer.java | 2 +
.../config/initialize/ModelBrokenListener.java | 12 +-
.../kylin/rest/service/KylinUserService.java | 8 +-
.../apache/kylin/rest/service/ProjectService.java | 5 +-
.../apache/kylin/rest/service/SystemService.java | 44 +++-
.../service/task/QueryHistoryTaskScheduler.java | 21 +-
.../task/RecommendationTopNUpdateScheduler.java | 16 +-
.../task/QueryHistoryTaskSchedulerTest.java | 17 +-
.../rest/util/InitResourceGroupUtilsTest.java} | 34 ++-
src/core-common/pom.xml | 9 +
.../org/apache/kylin/common/KylinConfigBase.java | 25 ++
.../common/exception/code/ErrorCodeSystem.java | 18 +-
.../common/persistence/InMemResourceStore.java | 7 +-
.../kylin/common/persistence/ResourceStore.java | 25 +-
.../common/persistence/RootPersistentEntity.java | 16 +-
.../persistence/ThreadViewResourceStore.java | 13 +-
.../persistence/lock/AbstractMemoryLock.java | 62 +++++
.../persistence/lock/DeadLockException.java} | 26 +--
.../common/persistence/lock/DeadLockHandler.java | 61 +++++
.../common/persistence/lock/DeadLockInfo.java | 72 ++++++
.../common/persistence/lock/IntentionLock.java} | 26 +--
.../persistence/lock/IntentionReadWriteLock.java | 257 +++++++++++++++++++++
.../persistence/lock/IntentionShareLock.java | 59 +++++
.../persistence/lock/LockInterruptException.java} | 26 +--
.../common/persistence/lock/LockManager.java} | 36 ++-
.../persistence/lock/LockManagerInstance.java} | 27 +--
.../persistence/lock/LockTimeoutException.java} | 26 +--
.../kylin/common/persistence/lock/MemoryLock.java} | 30 +--
.../common/persistence/lock/MemoryLockGraph.java | 171 ++++++++++++++
.../common/persistence/lock/MemoryLockManager.java | 97 ++++++++
.../common/persistence/lock/MemoryLockUtils.java | 206 +++++++++++++++++
.../kylin/common/persistence/lock/ModuleLock.java} | 42 ++--
.../common/persistence/lock/ModuleLockEnum.java | 87 +++++++
.../lock/OptimisticLockRetryException.java} | 25 +-
.../kylin/common/persistence/lock/PathLock.java} | 35 +--
.../common/persistence/lock/ProjectLock.java} | 36 +--
.../persistence/lock/ResourcePathParser.java | 121 ++++++++++
.../kylin/common/persistence/lock/TempLock.java} | 25 +-
.../lock/TransactionDeadLockHandler.java | 168 ++++++++++++++
.../common/persistence/lock/TransactionLock.java} | 29 +--
.../persistence/lock/TransactionLockManager.java} | 34 ++-
.../persistence/lock/TransactionProjectLock.java} | 26 +--
.../persistence/lock/TransactionTempLock.java} | 27 +--
.../lock/rule/LockEliminationRule.java} | 33 ++-
.../lock/rule/LockEliminationRuleExec.java | 57 +++++
.../common/persistence/lock/rule/LockInfo.java | 49 ++++
.../lock/rule/ModuleEliminationRule.java | 55 +++++
.../persistence/lock/rule/PathEliminationRule.java | 70 ++++++
.../lock/rule/ProjectEliminationRule.java} | 36 ++-
.../persistence/metadata/JdbcMetadataStore.java | 3 +
.../common/persistence/metadata/MetadataStore.java | 1 +
.../transaction/MessageSynchronization.java | 7 +-
.../persistence/transaction/TransactionLock.java | 96 --------
.../transaction/TransactionManagerInstance.java} | 32 ++-
.../TransactionOptimisticLockManager.java} | 26 +--
.../TransactionPessimisticLockManager.java | 81 +++++++
.../common/persistence/transaction/UnitOfWork.java | 163 +++++++++----
.../persistence/transaction/UnitOfWorkContext.java | 28 ++-
.../persistence/transaction/UnitOfWorkParams.java | 24 ++
.../org/apache/kylin/common/util/JsonUtil.java | 8 +
.../resources/kylin_error_msg_conf_cn.properties | 3 +
.../resources/kylin_error_msg_conf_en.properties | 3 +
.../main/resources/kylin_errorcode_conf.properties | 3 +
.../persistence/JdbcAuditLogRecoveryTest.java | 24 +-
.../common/persistence/JdbcMetadataStoreTest.java | 45 +++-
.../kylin/common/persistence/UnitOfWorkTest.java | 113 ++++++---
.../common/persistence/lock/CloseWaitTest.java | 167 +++++++++++++
.../persistence/lock/MemoryLockGraphTest.java | 218 +++++++++++++++++
.../lock/rule/LockEliminationRuleExecTest.java | 159 +++++++++++++
.../metadata/JdbcAuditLogStoreTest.java | 8 +-
...ckTest.java => TransactionProjectLockTest.java} | 13 +-
.../org/apache/kylin/job/common/SegmentUtil.java | 5 +-
.../org/apache/kylin/job/dao/JobStatistics.java | 10 +-
.../apache/kylin/job/dao/JobStatisticsBasic.java | 12 +-
.../apache/kylin/job/dao/JobStatisticsManager.java | 28 ++-
.../org/apache/kylin/job/dao/NExecutableDao.java | 20 +-
.../kylin/job/execution/NExecutableManager.java | 13 +-
.../kylin/metadata/TempStatementManager.java | 10 +-
.../java/org/apache/kylin/metadata/acl/AclTCR.java | 17 +-
.../apache/kylin/metadata/acl/AclTCRManager.java | 21 +-
.../metadata/cachesync/CacheReloadChecker.java | 6 +-
.../kylin/metadata/cachesync/CachedCrudAssist.java | 53 ++++-
.../kylin/metadata/cube/model/IndexPlan.java | 8 +
.../cube/model/NDataLoadingRangeManager.java | 111 +++++----
.../cube/model/NDataSegDetailsManager.java | 41 +++-
.../metadata/cube/model/NDataflowManager.java | 99 ++++----
.../metadata/cube/model/NIndexPlanManager.java | 38 +--
.../kylin/metadata/favorite/AbstractAsyncTask.java | 11 +
.../metadata/favorite/AsyncAccelerationTask.java | 3 +-
.../kylin/metadata/favorite/AsyncTaskManager.java | 30 ++-
.../kylin/metadata/favorite/FavoriteRule.java | 17 +-
.../metadata/favorite/FavoriteRuleManager.java | 22 +-
.../metadata/favorite/QueryHistoryIdOffset.java | 11 +
.../favorite/QueryHistoryIdOffsetManager.java | 25 +-
.../apache/kylin/metadata/jar/JarInfoManager.java | 34 ++-
.../kylin/metadata/model/ComputedColumnDesc.java | 4 +
.../apache/kylin/metadata/model/FusionModel.java | 10 +-
.../kylin/metadata/model/FusionModelManager.java | 11 +-
.../apache/kylin/metadata/model/NDataModel.java | 24 ++
.../kylin/metadata/model/NDataModelManager.java | 31 ++-
.../metadata/model/NTableMetadataManager.java | 115 ++++++---
.../org/apache/kylin/metadata/model/TableDesc.java | 10 +-
.../apache/kylin/metadata/model/TableExtDesc.java | 19 +-
.../metadata/model/schema/ImportModelContext.java | 2 +-
.../kylin/metadata/project/NProjectManager.java | 12 +-
.../resourcegroup/ResourceGroupManager.java | 2 +-
.../metadata/sourceusage/SourceUsageManager.java | 9 +-
.../metadata/streaming/DataParserManager.java | 20 +-
.../metadata/streaming/KafkaConfigManager.java | 17 +-
.../kylin/metadata/streaming/StreamingManager.java | 13 +-
.../metadata/upgrade/GlobalAclVersionManager.java | 10 +-
.../apache/kylin/metadata/user/ManagedUser.java | 7 +
.../kylin/metadata/user/NKylinUserManager.java | 37 ++-
.../metadata/usergroup/NUserGroupManager.java | 4 +-
.../apache/kylin/metadata/usergroup/UserGroup.java | 7 +
.../org/apache/kylin/rest/security/AclManager.java | 18 +-
.../org/apache/kylin/rest/security/AclRecord.java | 2 +-
.../org/apache/kylin/rest/security/UserAcl.java | 7 +
.../apache/kylin/rest/security/UserAclManager.java | 4 +-
.../org/apache/kylin/util/BrokenEntityProxy.java | 10 +-
.../metadata/cachesync/CachedCrudAssistTest.java | 63 +++++
.../apache/kylin/metadata/cube/IndexPlanTest.java | 49 ++--
.../cube/model/NDataLoadingRangeManagerTest.java | 28 +--
.../cube/model/NDataSegDetailsManagerTest.java | 2 +-
.../metadata/epoch/EnhancedUnitOfWorkTest.java | 8 +-
.../metadata/model/NDataModelManagerTest.java | 50 ++++
.../metadata/project/NProjectManagerTest.java | 32 +++
.../org/apache/kylin/model/NDataModelTest.java | 53 +++++
.../sdk/datasource/PushdownRunnerSDKImplTest.java | 10 +-
.../apache/kylin/rest/service/TableExtService.java | 2 +-
.../apache/kylin/rest/service/TableService.java | 114 +++++----
.../rest/service/StreamingTableServiceTest.java | 4 -
.../e78a89dd-847f-4574-8afa-8768b4228b72.json | 1 -
.../e78a89dd-847f-4574-8afa-8768b4228b73.json | 1 -
.../e78a89dd-847f-4574-8afa-8768b4228b74.json | 1 -
.../org/apache/kylin/newten/AutoMergeTest.java | 7 +-
.../org/apache/kylin/event/HAMetadataTest.java | 23 +-
.../kylin/rest/controller/NTableController.java | 5 +-
.../apache/kylin/rest/request/ModelRequest.java | 4 +
.../apache/kylin/rest/service/ModelService.java | 160 ++++++-------
.../kylin/rest/service/IndexPlanServiceTest.java | 13 +-
.../kylin/rest/service/ModelServiceTest.java | 3 +-
.../kylin/rest/service/TableReloadServiceTest.java | 8 +-
.../kylin/rest/service/TableServiceTest.java | 4 +-
.../apache/kylin/rest/service/QueryService.java | 23 +-
.../kylin/rest/service/QueryServiceTest.java | 22 ++
.../rest/util/QueryCacheSignatureUtilTest.java | 16 +-
.../apache/kylin/query/blacklist/SQLBlacklist.java | 13 +-
.../kylin/query/blacklist/SQLBlacklistManager.java | 28 ++-
.../rest/health/MetaStoreHealthIndicator.java | 3 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 28 ++-
.../engine/spark/job/NSparkUpdateMetadataStep.java | 6 +-
.../spark/builder/SnapshotPartitionBuilder.scala | 6 +-
.../kylin/engine/spark/job/JobManagerTest.java | 5 +-
.../query/pushdown/PushDownRunnerJdbcImplTest.java | 56 ++---
.../streaming/manager/StreamingJobManager.java | 9 +-
.../apache/kylin/helper/MetadataToolHelper.java | 21 +-
.../apache/kylin/tool/garbage/StorageCleaner.java | 22 +-
.../kylin/tool/security/KylinPasswordResetCLI.java | 10 +-
.../kylin/tool/upgrade/UpdateProjectCLI.java | 29 +--
.../util/ProjectTemporaryTableCleanerHelper.java | 4 +-
.../org/apache/kylin/tool/AuditLogToolTest.java | 11 +-
.../apache/kylin/tool/MetadataToolTestFixture.java | 2 +-
.../apache/kylin/tool/YarnApplicationToolTest.java | 11 +-
.../tool/security/KylinPasswordResetCLITest.java | 11 +-
.../ProjectTemporaryTableCleanerHelperTest.java | 5 +-
172 files changed, 4427 insertions(+), 1348 deletions(-)
diff --git a/build/bin/create-table.sh b/build/bin/create-table.sh
new file mode 100755
index 0000000000..5fbc49541d
--- /dev/null
+++ b/build/bin/create-table.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z $KYLIN_HOME ];then
+ export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
+fi
+
+function help {
+ echo "usage: create-table.sh table_path (the table path like /metadata_backup_path/project_name/table/)"
+ exit 1
+}
+
+if [ $# -ne 1 ]; then
+ help
+else
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.util.CreateTableFromJson $1
+fi
diff --git a/build/conf/kylin-tools-log4j.xml b/build/conf/kylin-tools-log4j.xml
index e5e40d0beb..2abe924057 100644
--- a/build/conf/kylin-tools-log4j.xml
+++ b/build/conf/kylin-tools-log4j.xml
@@ -20,6 +20,9 @@
<Console name="stderr" target="SYSTEM_ERR">
<PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p %X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
+ <File name="createTableSql" fileName="${env:KYLIN_HOME}/logs/create_table.sql">
+ <PatternLayout pattern="%msg%n"/>
+ </File>
</Appenders>
<Loggers>
<Root level="INFO">
@@ -32,5 +35,8 @@
<Logger name="io.kyligence" level="DEBUG"/>
<Logger name="org.apache.kylin.tool.shaded" level="INFO"/>
<Logger name="org.apache.kylin" level="DEBUG"/>
+ <Logger name="org.apache.kylin.tool.util.CreateTableFromJson" level="INFO">
+ <AppenderRef ref="createTableSql"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
index 83a5d75211..5e14ee75e7 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
@@ -28,11 +28,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
// CALL FROM CORE
@NotThreadSafe
@@ -69,6 +67,10 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
this.crud.setCheckCopyOnWrite(true);
}
+ public String getResourcePath(String resourceName) {
+ return crud.resourcePath(resourceName);
+ }
+
@SuppressWarnings("unchecked")
protected T save(T entity) {
if (entity instanceof IManagerAware) {
@@ -81,6 +83,10 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
protected T copy(T entity) {
return crud.copyBySerialization(entity);
}
+
+ protected T copyForWrite(T entity) {
+ return crud.copyForWrite(entity);
+ }
// Create
protected abstract T newRootEntity(String cubeName);
@@ -92,10 +98,11 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
public T createAS(T entity) {
if (entity.getUuid() == null)
throw new IllegalArgumentException();
- if (crud.contains(entity.getUuid()))
+ T copy = copyForWrite(entity);
+ if (crud.contains(copy.getUuid()))
throw new IllegalArgumentException("Entity '" + entity.getUuid() + "' already exists");
// overwrite point
- return save(entity);
+ return save(copy);
}
// Read
@@ -126,14 +133,14 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
}
public T update(String uuid, Consumer<T> updater) {
- return get(uuid).map(this::copy).map(copied -> {
+ return get(uuid).map(this::copyForWrite).map(copied -> {
updater.accept(copied);
return internalUpdate(copied);
}).orElse(null);
}
protected T upsert(String uuid, Consumer<T> updater, Supplier<T> creator) {
- return get(uuid).map(this::copy).map(copied -> {
+ return get(uuid).map(this::copyForWrite).map(copied -> {
updater.accept(copied);
return internalUpdate(copied);
}).orElseGet(() -> {
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
index b915326ba1..d328d43104 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
@@ -81,6 +81,11 @@ public class NodeGroup extends RootPersistentEntity implements Serializable,
return this;
}
+ @Override
+ public String getResourcePath() {
+ return manager.getResourcePath(resourceName());
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
index ea1fe04af2..c6eb1838a6 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
@@ -145,6 +145,11 @@ public class TablePlan extends RootPersistentEntity
return description;
}
+ @Override
+ public String getResourcePath() {
+ return manager.getResourcePath(resourceName());
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
index b29d4a4eaf..50fe881663 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
@@ -26,6 +26,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.TIME_INVALI
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -38,6 +39,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.persistence.lock.DeadLockInfo;
import org.apache.kylin.common.persistence.transaction.EpochCheckBroadcastNotifier;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
@@ -390,4 +392,35 @@ public class NSystemController extends NBasicController {
setDownloadResponse(backupInputStream, METADATA_FILE, MediaType.APPLICATION_OCTET_STREAM_VALUE, response);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, "", "");
}
+
+ @PostMapping(value = "/deadlock/detect")
+ @ResponseBody
+ public EnvelopeResponse<List<DeadLockInfo>> detectDeadLock() {
+ return new EnvelopeResponse<>(CODE_SUCCESS, systemService.detectDeadLock(), "");
+ }
+
+ @PostMapping(value = "/deadlock/kill")
+ @ResponseBody
+ public EnvelopeResponse<String> killThread(@RequestParam(value = "ids") List<Long> ids) {
+ checkCollectionRequiredArg("ids", ids);
+ systemService.killDeadLockThread(ids);
+ return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
+ }
+
+ @PostMapping(value = "/deadlock/kill_all")
+ @ResponseBody
+ public EnvelopeResponse<String> killAllThread() {
+ systemService.killAllDeadLockThread();
+ return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
+ }
+
+ /**
+ * RPC Call
+ */
+ @DeleteMapping(value = "clean_sparder_event_log")
+ @ResponseBody
+ public EnvelopeResponse<String> queryNodeCleanSparderEventsLogs() {
+ systemService.cleanSparderEventLog();
+ return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
+ }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
index c58b5169fc..f280d27509 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
@@ -29,6 +29,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.Constant;
import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.lock.TransactionDeadLockHandler;
import org.apache.kylin.common.persistence.metadata.EpochStore;
import org.apache.kylin.common.persistence.metadata.JdbcAuditLogStore;
import org.apache.kylin.common.persistence.transaction.EventListenerRegistry;
@@ -135,6 +136,7 @@ public class AppInitializer {
EventBusFactory.getInstance().register(new StreamingJobListener(), true);
SparkJobFactoryUtils.initJobFactory();
+ TransactionDeadLockHandler.getInstance().start();
} else {
val auditLogStore = new JdbcAuditLogStore(kylinConfig);
val epochStore = EpochStore.getEpochStore(kylinConfig);
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java
index 3dece15a36..b433cf43ad 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
import org.apache.kylin.job.manager.JobManager;
import org.apache.kylin.job.model.JobParam;
import org.apache.kylin.metadata.cube.model.NDataSegment;
@@ -41,9 +43,6 @@ import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
import org.apache.kylin.metadata.sourceusage.SourceUsageManager;
import org.springframework.util.CollectionUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -94,9 +93,10 @@ public class ModelBrokenListener {
dfUpdate.setToRemoveSegs(dataflow.getSegments().toArray(new NDataSegment[0]));
}
dataflowManager.updateDataflow(dfUpdate);
- model.setHandledAfterBroken(true);
- model.setRecommendationsCount(0);
- modelManager.updateDataBrokenModelDesc(model);
+ NDataModel copy = modelManager.copyForWrite(model);
+ copy.setHandledAfterBroken(true);
+ copy.setRecommendationsCount(0);
+ modelManager.updateDataBrokenModelDesc(copy);
OptRecManagerV2 optRecManagerV2 = OptRecManagerV2.getInstance(project);
optRecManagerV2.discardAll(model.getId());
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java
index c9d1b69e9a..de0dfeecae 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java
@@ -36,6 +36,7 @@ import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -47,8 +48,6 @@ import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.util.CollectionUtils;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.metadata.user.NKylinUserManager;
import lombok.SneakyThrows;
@@ -126,8 +125,9 @@ public class KylinUserService implements UserService {
public UserDetails loadUserByUsername(String userName) throws UsernameNotFoundException {
Message msg = MsgPicker.getMsg();
ManagedUser managedUser;
+ NKylinUserManager manager = getKylinUserManager();
try {
- managedUser = getKylinUserManager().get(userName);
+ managedUser = manager.get(userName);
} catch (IllegalArgumentException e) {
log.error("exception: ", e);
throw new UsernameNotFoundException(USER_LOGIN_FAILED.getMsg());
@@ -136,7 +136,7 @@ public class KylinUserService implements UserService {
throw new UsernameNotFoundException(String.format(Locale.ROOT, msg.getUserNotFound(), userName));
}
log.trace("load user : {}", userName);
- return managedUser;
+ return manager.copy(managedUser);
}
@Override
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 4299f0d7e3..95c7578f22 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -19,13 +19,13 @@
package org.apache.kylin.rest.service;
import static org.apache.kylin.common.constant.Constants.HIDDEN_VALUE;
+import static org.apache.kylin.common.constant.Constants.KYLIN_JOB_MAX_CONCURRENT_JOBS;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_DRIVER_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_ENABLE_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_NAME_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_USER_KEY;
-import static org.apache.kylin.common.constant.Constants.KYLIN_JOB_MAX_CONCURRENT_JOBS;
import static org.apache.kylin.common.constant.NonCustomProjectLevelConfig.DATASOURCE_TYPE;
import static org.apache.kylin.common.exception.ServerErrorCode.DATABASE_NOT_EXIST;
import static org.apache.kylin.common.exception.ServerErrorCode.DUPLICATE_PROJECT_NAME;
@@ -970,8 +970,7 @@ public class ProjectService extends BasicService {
if (uppderDB.equals(projectInstance.getDefaultDatabase())) {
return;
}
- projectInstance.setDefaultDatabase(uppderDB);
- prjManager.updateProject(projectInstance);
+ prjManager.updateProject(project, copyForWrite -> copyForWrite.setDefaultDatabase(uppderDB));
} else {
throw new KylinException(DATABASE_NOT_EXIST,
String.format(Locale.ROOT, MsgPicker.getMsg().getDatabaseNotExist(), defaultDatabase));
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
index e983bcb46c..59cf7d5543 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
@@ -22,6 +22,7 @@ import static org.apache.kylin.common.exception.ServerErrorCode.CONFIG_NONEXIST_
import static org.apache.kylin.common.exception.ServerErrorCode.DIAG_FAILED;
import static org.apache.kylin.common.exception.ServerErrorCode.DIAG_UUID_NOT_EXIST;
import static org.apache.kylin.common.exception.ServerErrorCode.FILE_NOT_EXIST;
+import static org.apache.kylin.common.exception.code.ErrorCodeSystem.NOT_DEADLOCK_THREAD_IDS;
import static org.apache.kylin.tool.constant.DiagTypeEnum.FULL;
import static org.apache.kylin.tool.constant.DiagTypeEnum.JOB;
import static org.apache.kylin.tool.constant.DiagTypeEnum.QUERY;
@@ -29,10 +30,14 @@ import static org.apache.kylin.tool.constant.StageEnum.DONE;
import java.io.File;
import java.io.IOException;
+import java.lang.management.ThreadInfo;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -42,6 +47,7 @@ import java.util.stream.Collectors;
import javax.validation.constraints.NotNull;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -49,11 +55,16 @@ import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.KylinTimeoutException;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.persistence.lock.DeadLockInfo;
+import org.apache.kylin.common.persistence.lock.TransactionDeadLockHandler;
import org.apache.kylin.common.persistence.transaction.MessageSynchronization;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.guava30.shaded.common.cache.Cache;
+import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -79,9 +90,6 @@ import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
-import org.apache.kylin.guava30.shaded.common.cache.Cache;
-import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
-
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.val;
@@ -115,7 +123,8 @@ public class SystemService extends BasicService {
}
}
- private final Cache<String, DiagInfo> diagMap = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS).build();
+ private final Cache<String, DiagInfo> diagMap = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS)
+ .build();
private final Cache<String, DiagStatusResponse> exceptionMap = CacheBuilder.newBuilder()
.expireAfterAccess(1, TimeUnit.DAYS).build();
private final ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -123,7 +132,7 @@ public class SystemService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#backupRequest.getProject(), 'ADMINISTRATION')")
public void backup(BackupRequest backupRequest) throws Exception {
String project = StringUtils.isNotBlank(backupRequest.getProject()) ? backupRequest.getProject() : null;
- String path = StringUtils.isNotBlank(backupRequest.getBackupPath()) ? backupRequest.getBackupPath(): null;
+ String path = StringUtils.isNotBlank(backupRequest.getBackupPath()) ? backupRequest.getBackupPath() : null;
boolean compress = backupRequest.isCompress();
helper.backup(getConfig(), project, path, null, compress, false);
}
@@ -395,4 +404,29 @@ public class SystemService extends BasicService {
RoutineToolHelper.cleanEventLog(true, true, false);
}
}
+
+ public List<DeadLockInfo> detectDeadLock() {
+ Set<ThreadInfo> threadInfoSet = TransactionDeadLockHandler.getInstance().getThreadsToBeKill();
+ if (CollectionUtils.isEmpty(threadInfoSet)) {
+ return Collections.emptyList();
+ }
+ return threadInfoSet.stream().map(DeadLockInfo::new).collect(Collectors.toList());
+ }
+
+ public void killDeadLockThread(List<Long> ids) {
+ TransactionDeadLockHandler deadLockHandler = TransactionDeadLockHandler.getInstance();
+
+ Set<Long> threadIdSet = Sets.newHashSet(ids);
+ threadIdSet.removeAll(deadLockHandler.getThreadIdToBeKill());
+ if (CollectionUtils.isNotEmpty(threadIdSet)) {
+ throw new KylinException(NOT_DEADLOCK_THREAD_IDS, StringUtils.join(threadIdSet, ","));
+ }
+ deadLockHandler.killThreadsById(Sets.newHashSet(ids));
+ }
+
+ public void killAllDeadLockThread() {
+ TransactionDeadLockHandler deadLockHandler = TransactionDeadLockHandler.getInstance();
+ Set<Long> threadIdToBeKill = deadLockHandler.getThreadIdToBeKill();
+ deadLockHandler.killThreadsById(threadIdToBeKill);
+ }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java
index 86e5d286ba..386c8791ff 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java
@@ -137,9 +137,10 @@ public class QueryHistoryTaskScheduler {
QueryHistoryIdOffsetManager manager = QueryHistoryIdOffsetManager.getInstance(config, project);
QueryHistoryIdOffset queryHistoryIdOffset = manager.get();
if (queryHistoryIdOffset.getOffset() > maxId || queryHistoryIdOffset.getStatMetaUpdateOffset() > maxId) {
- queryHistoryIdOffset.setOffset(maxId);
- queryHistoryIdOffset.setStatMetaUpdateOffset(maxId);
- manager.save(queryHistoryIdOffset);
+ manager.updateOffset(copyForWrite -> {
+ copyForWrite.setOffset(maxId);
+ copyForWrite.setStatMetaUpdateOffset(maxId);
+ });
}
return 0;
}, project);
@@ -230,10 +231,9 @@ public class QueryHistoryTaskScheduler {
updateLastQueryTime(modelsLastQueryTime, project);
// update id offset
- QueryHistoryIdOffset queryHistoryIdOffset = QueryHistoryIdOffsetManager
- .getInstance(KylinConfig.getInstanceFromEnv(), project).get();
- queryHistoryIdOffset.setStatMetaUpdateOffset(maxId);
- QueryHistoryIdOffsetManager.getInstance(config, project).save(queryHistoryIdOffset);
+ QueryHistoryIdOffsetManager manager = QueryHistoryIdOffsetManager
+ .getInstance(config, project);
+ manager.updateOffset(copyForWrite -> copyForWrite.setStatMetaUpdateOffset(maxId));
// update snpashot hit count
incQueryHitSnapshotCount(hitSnapshotCountMap, project);
@@ -424,10 +424,9 @@ public class QueryHistoryTaskScheduler {
KylinConfig config = KylinConfig.getInstanceFromEnv();
// update id offset
- QueryHistoryIdOffset queryHistoryIdOffset = QueryHistoryIdOffsetManager
- .getInstance(KylinConfig.getInstanceFromEnv(), project).get();
- queryHistoryIdOffset.setOffset(maxId);
- QueryHistoryIdOffsetManager.getInstance(config, project).save(queryHistoryIdOffset);
+ QueryHistoryIdOffsetManager manager = QueryHistoryIdOffsetManager
+ .getInstance(config, project);
+ manager.updateOffset(copyForWrite -> copyForWrite.setOffset(maxId));
return 0;
}, project);
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java
index 6e590b6f66..044ef27ac1 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java
@@ -29,14 +29,15 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.metrics.MetricsCategory;
+import org.apache.kylin.common.metrics.MetricsGroup;
+import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.util.ExecutorServiceUtil;
import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.common.util.TimeUtil;
-import org.apache.kylin.common.metrics.MetricsCategory;
-import org.apache.kylin.common.metrics.MetricsGroup;
-import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.metadata.favorite.AsyncAccelerationTask;
+import org.apache.kylin.metadata.favorite.AsyncTaskManager;
import org.apache.kylin.metadata.favorite.FavoriteRule;
import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
@@ -146,10 +147,11 @@ public class RecommendationTopNUpdateScheduler {
public void saveTaskTime(String project) {
long currentTime = System.currentTimeMillis();
EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) getInstance(
- KylinConfig.getInstanceFromEnv(), project).get(ASYNC_ACCELERATION_TASK);
- asyncAcceleration.setLastUpdateTonNTime(currentTime);
- getInstance(KylinConfig.getInstanceFromEnv(), project).save(asyncAcceleration);
+ AsyncTaskManager manager = getInstance(KylinConfig.getInstanceFromEnv(), project);
+ AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) manager.get(ASYNC_ACCELERATION_TASK);
+ AsyncAccelerationTask copied = manager.copyForWrite(asyncAcceleration);
+ copied.setLastUpdateTonNTime(currentTime);
+ getInstance(KylinConfig.getInstanceFromEnv(), project).save(copied);
return null;
}, project);
}
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
index 9e70136557..a505364ef3 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
@@ -33,7 +33,6 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.favorite.AccelerateRuleUtil;
import org.apache.kylin.metadata.favorite.AsyncAccelerationTask;
import org.apache.kylin.metadata.favorite.AsyncTaskManager;
-import org.apache.kylin.metadata.favorite.QueryHistoryIdOffset;
import org.apache.kylin.metadata.favorite.QueryHistoryIdOffsetManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -144,10 +143,10 @@ public class QueryHistoryTaskSchedulerTest extends NLocalFileMetadataTestCase {
// before update id offset
QueryHistoryIdOffsetManager idOffsetManager = QueryHistoryIdOffsetManager.getInstance(getTestConfig(), PROJECT);
- QueryHistoryIdOffset queryHistoryIdOffset = idOffsetManager.get();
- queryHistoryIdOffset.setOffset(999L);
- queryHistoryIdOffset.setStatMetaUpdateOffset(999L);
- idOffsetManager.save(queryHistoryIdOffset);
+ idOffsetManager.updateOffset(copyForWrite -> {
+ copyForWrite.setOffset(999L);
+ copyForWrite.setStatMetaUpdateOffset(999L);
+ });
Assert.assertEquals(999L, idOffsetManager.get().getOffset());
// run update
qhAccelerateScheduler.resetOffsetId();
@@ -164,10 +163,10 @@ public class QueryHistoryTaskSchedulerTest extends NLocalFileMetadataTestCase {
Mockito.when(qhAccelerateScheduler.queryHistoryDAO.getQueryHistoryMaxId(Mockito.anyString())).thenReturn(12L);
// before update id offset
QueryHistoryIdOffsetManager idOffsetManager = QueryHistoryIdOffsetManager.getInstance(getTestConfig(), PROJECT);
- QueryHistoryIdOffset queryHistoryIdOffset = idOffsetManager.get();
- queryHistoryIdOffset.setOffset(9L);
- queryHistoryIdOffset.setStatMetaUpdateOffset(9L);
- idOffsetManager.save(queryHistoryIdOffset);
+ idOffsetManager.updateOffset(copyForWrite -> {
+ copyForWrite.setOffset(9L);
+ copyForWrite.setStatMetaUpdateOffset(9L);
+ });
Assert.assertEquals(9L, idOffsetManager.get().getOffset());
// run update
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java
similarity index 53%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java
index 1bbd33831f..b6ae0f1f0e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java
@@ -16,27 +16,23 @@
* limitations under the License.
*/
-package org.apache.kylin.metadata.favorite;
+package org.apache.kylin.rest.util;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.resourcegroup.ResourceGroupManager;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
-import com.fasterxml.jackson.annotation.JsonProperty;
+@MetadataInfo(onlyProps = true)
+public class InitResourceGroupUtilsTest {
+
+ @Test
+ void testInitResourceGroup() {
+ ResourceGroupManager manager = ResourceGroupManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assertions.assertFalse(manager.resourceGroupInitialized());
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ InitResourceGroupUtils.initResourceGroup();
+ Assertions.assertTrue(manager.resourceGroupInitialized());
}
}
diff --git a/src/core-common/pom.xml b/src/core-common/pom.xml
index 178418deac..a7f1426270 100644
--- a/src/core-common/pom.xml
+++ b/src/core-common/pom.xml
@@ -80,6 +80,10 @@
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-common</artifactId>
+ </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>transmittable-thread-local</artifactId>
@@ -186,6 +190,11 @@
<artifactId>jcl-over-slf4j</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f51bb8ba8a..954d13a685 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -700,6 +700,31 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.env.ip-address");
}
+ public String getTransactionLockManager() {
+ return getOptional("kylin.env.transaction-lock-manager",
+ "org.apache.kylin.common.persistence.transaction.TransactionOptimisticLockManager");
+ }
+
+ public String getLockManager() {
+ return getOptional("kylin.env.lock-manager", "org.apache.kylin.common.persistence.lock.MemoryLockManager");
+ }
+
+ public long getLockTimeoutSeconds() {
+ return Long.parseLong(getOptional("kylin.env.lock-timeout", "60"));
+ }
+
+ public long getLockCheckIntervalSeconds() {
+ return Long.parseLong(getOptional("kylin.env.dead-lock-check-interval", "3"));
+ }
+
+ public long getMaxSecondsForDeadLockRetry() {
+ return Long.parseLong(getOptional("kylin.env.max-seconds-for-dead-lock-retry", "600"));
+ }
+
+ public int getLockRemoveMaxSize() {
+ return Integer.parseInt(getOptional("kylin.env.lock-remove-max-size", "200"));
+ }
+
public String getServerAddress() {
// Caution: config 'kylin.server.address' is essential in yarn cluster mode.
// The value may be the address of loadbalancer
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java
index 12b057c677..202ea66e6c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java
@@ -20,21 +20,27 @@ package org.apache.kylin.common.exception.code;
public enum ErrorCodeSystem implements ErrorCodeProducer {
// 400052XX password
- PASSWORD_INVALID_ENCODER("KE-040005201"), PASSWORD_INIT_ENCODER_FAILED("KE-040005202"),
+ PASSWORD_INVALID_ENCODER("KE-040005201"),
+ PASSWORD_INIT_ENCODER_FAILED("KE-040005202"),
// 400212XX epoch
EPOCH_DOES_NOT_BELONG_TO_CURRENT_NODE("KE-040021201"),
// 400232XX HA
- JOB_NODE_API_INVALID("KE-040023201"), JOB_NODE_QUERY_API_INVALID("KE-040023202"), QUERY_NODE_API_INVALID(
- "KE-040023203"),
+ JOB_NODE_API_INVALID("KE-040023201"),
+ JOB_NODE_QUERY_API_INVALID("KE-040023202"),
+ QUERY_NODE_API_INVALID("KE-040023203"),
// 400242XX maintenance mode
- MAINTENANCE_MODE_WRITE_FAILED("KE-040024201"), MAINTENANCE_MODE_ENTER_FAILED(
- "KE-040024202"), MAINTENANCE_MODE_LEAVE_FAILED("KE-040024203"),
+ MAINTENANCE_MODE_WRITE_FAILED("KE-040024201"),
+ MAINTENANCE_MODE_ENTER_FAILED("KE-040024202"),
+ MAINTENANCE_MODE_LEAVE_FAILED("KE-040024203"),
// 400252XX system config
- SYSTEM_PROFILE_ABNORMAL_DATA("KE-040026201");
+ SYSTEM_PROFILE_ABNORMAL_DATA("KE-040026201"),
+
+ // 400282XX deadlock
+ NOT_DEADLOCK_THREAD_IDS("KE-040028201");
private final ErrorCode errorCode;
private final ErrorMsg errorMsg;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java
index c2b87f1ace..26ce0935c2 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java
@@ -24,15 +24,14 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.FilePathUtil;
import org.apache.kylin.common.persistence.metadata.MetadataStore.MemoryMetaData;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.FilePathUtil;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.Getter;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index c720f29254..b057300025 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -41,21 +41,21 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.AuditLogStore;
import org.apache.kylin.common.persistence.metadata.EpochStore;
import org.apache.kylin.common.persistence.metadata.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.guava30.shaded.common.base.Throwables;
import org.apache.kylin.guava30.shaded.common.cache.Cache;
import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
import org.apache.kylin.guava30.shaded.common.cache.CacheLoader;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import lombok.Getter;
import lombok.Setter;
import lombok.val;
@@ -223,6 +223,11 @@ public abstract class ResourceStore implements AutoCloseable {
* Read a resource, return null in case of not found or is a folder.
*/
public final <T extends RootPersistentEntity> T getResource(String resPath, Serializer<T> serializer) {
+ return MemoryLockUtils.doWithLock(null, resPath, true, this,
+ () -> getResourceWithoutLock(resPath, serializer));
+ }
+
+ public final <T extends RootPersistentEntity> T getResourceWithoutLock(String resPath, Serializer<T> serializer) {
resPath = norm(resPath);
RawResource res = getResourceImpl(resPath);
if (res == null)
@@ -244,7 +249,7 @@ public abstract class ResourceStore implements AutoCloseable {
}
public final RawResource getResource(String resPath) {
- return getResourceImpl(norm(resPath));
+ return MemoryLockUtils.doWithLock(resPath, true, this, () -> getResourceImpl(norm(resPath)));
}
public final List<RawResource> getAllResources(String folderPath) {
@@ -484,8 +489,9 @@ public abstract class ResourceStore implements AutoCloseable {
try {
File f = Paths.get(metaDir.getAbsolutePath(), res.getResPath()).toFile();
f.getParentFile().mkdirs();
- try (FileOutputStream out = new FileOutputStream(f)) {
- IOUtils.copy(res.getByteSource().openStream(), out);
+ try (FileOutputStream out = new FileOutputStream(f);
+ InputStream in = res.getByteSource().openStream()) {
+ IOUtils.copy(in, out);
if (!f.setLastModified(res.getTimestamp())) {
logger.info("{} modified time change failed", f);
}
@@ -567,6 +573,7 @@ public abstract class ResourceStore implements AutoCloseable {
public void createMetaStoreUuidIfNotExist() {
if (!exists(METASTORE_UUID_TAG)) {
+ MemoryLockUtils.doWithLock(null, METASTORE_UUID_TAG, false, this, () -> null);
checkAndPutResource(METASTORE_UUID_TAG, new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index d939ce6691..cde9e01b10 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -19,17 +19,19 @@
package org.apache.kylin.common.persistence;
import java.io.Serializable;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.springframework.beans.BeanUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
@@ -149,6 +151,10 @@ public abstract class RootPersistentEntity implements AclEntity, Serializable {
}
this.mvcc = mvcc;
}
+
+ public <T extends RootPersistentEntity> void copyPropertiesTo(T copy) {
+ BeanUtils.copyProperties(this, copy, "cachedAndShared");
+ }
/**
* The name as a part of the resource path used to save the entity.
@@ -192,6 +198,14 @@ public abstract class RootPersistentEntity implements AclEntity, Serializable {
return "";
}
+ public List<String> getLockPaths() {
+ return getLockPaths(getResourcePath());
+ }
+
+ public List<String> getLockPaths(String resourcePath) {
+ return Collections.singletonList(resourcePath);
+ }
+
@Override
public String toString() {
return getClass().getName() + "@" + uuid;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java
index e847e92c30..85ef794d8e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java
@@ -25,12 +25,13 @@ import java.util.TreeSet;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.metadata.MetadataStore;
import org.apache.kylin.common.util.FilePathUtil;
-
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+
import lombok.Getter;
import lombok.val;
@@ -54,6 +55,11 @@ public class ThreadViewResourceStore extends ResourceStore {
this.overlay = new InMemResourceStore(kylinConfig);
resources = Lists.newArrayList();
}
+
+ @Override
+ public MetadataStore getMetadataStore() {
+ return underlying.getMetadataStore();
+ }
@Override
protected NavigableSet<String> listResourcesImpl(String folderPath, boolean recursive) {
@@ -106,9 +112,7 @@ public class ThreadViewResourceStore extends ResourceStore {
return r == TombRawResource.getINSTANCE() ? null //deleted
: r; // updated
}
-
return underlying.getResourceImpl(resPath);
-
}
@Override
@@ -142,6 +146,7 @@ public class ThreadViewResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) {
+ MemoryLockUtils.doWithLock(resPath, false, this, () -> null);
overlay.putTomb(resPath);
resources.add(new TombRawResource(resPath));
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java
new file mode 100644
index 0000000000..3567d92c28
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public abstract class AbstractMemoryLock extends IntentionReadWriteLock {
+
+ private final AtomicLong tranHoldCount = new AtomicLong();
+
+ private final Long createTime;
+
+ AbstractMemoryLock(IntentionShareLock intentionShareLock) {
+ super(intentionShareLock);
+ createTime = System.currentTimeMillis();
+ }
+
+ AbstractMemoryLock(IntentionReadWriteLock intentionReadWriteLock) {
+ super(new IntentionShareLock(intentionReadWriteLock));
+ createTime = System.currentTimeMillis();
+ }
+
+ @Override
+ public Long createTime() {
+ return createTime;
+ }
+
+ public AtomicLong getTranHoldCount() {
+ return tranHoldCount;
+ }
+
+ @Override
+ public TransactionLock readLock() {
+ tranHoldCount.incrementAndGet();
+ return super.readLock();
+ }
+
+ @Override
+ public TransactionLock writeLock() {
+ tranHoldCount.incrementAndGet();
+ return super.writeLock();
+ }
+
+ public TransactionLock getReadOrWriteLock(boolean read) {
+ return read ? readLock() : writeLock();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java
index 1bbd33831f..7e3c2d9002 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+public class DeadLockException extends RuntimeException {
+ public DeadLockException(String msg) {
+ super(msg);
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java
new file mode 100644
index 0000000000..636f728368
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.lang.management.ThreadInfo;
+import java.util.Set;
+
+public interface DeadLockHandler {
+
+ /**
+ * Detect deadlock ThreadInfo
+ */
+ ThreadInfo[] findDeadLockThreads();
+
+ /**
+ * Get the ThreadInfo that needs to be killed
+ */
+ Set<ThreadInfo> getThreadsToBeKill();
+
+ /**
+ * Get the ThreadIds that needs to be killed
+ */
+ Set<Long> getThreadIdToBeKill();
+
+ /**
+ * Filter thread based on ThreadId
+ */
+ Set<Thread> getThreadsById(Set<Long> threadIdSet);
+
+ /**
+ * Interrupt threads
+ */
+ void killThreads(Set<Thread> threads);
+
+ /**
+ * Interrupt threads by threadIds
+ */
+ void killThreadsById(Set<Long> threadIdSet);
+
+ /**
+ * Thread name prefix for filtering
+ */
+ String getThreadNamePrefix();
+
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java
new file mode 100644
index 0000000000..b43fcc71da
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.lang.management.LockInfo;
+import java.lang.management.ThreadInfo;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Data;
+
+@Data
+public class DeadLockInfo {
+
+ @JsonProperty("thread_id")
+ private long threadId;
+ @JsonProperty("thread_name")
+ private String threadName;
+ @JsonProperty("owned_locks")
+ private List<String> ownedLocks;
+ @JsonProperty("waiting_lock")
+ private String waitingLock;
+ @JsonProperty("stacktrace")
+ private String stacktrace;
+
+ @JsonIgnore
+ private static final int MAX_STACKTRACE_LENGTH = 16;
+
+ public DeadLockInfo(ThreadInfo ti) {
+ this.stacktrace = getStackTraceString(ti.getStackTrace());
+ this.threadName = ti.getThreadName();
+ this.threadId = ti.getThreadId();
+ this.waitingLock = ti.getLockInfo().toString();
+ this.ownedLocks = Arrays.stream(ti.getLockedSynchronizers()).map(LockInfo::toString)
+ .collect(Collectors.toList());
+ }
+
+ private String getStackTraceString(StackTraceElement[] st) {
+ StringBuilder sb = new StringBuilder();
+ int i = 0;
+ for (; i < st.length && i < MAX_STACKTRACE_LENGTH; i++) {
+ StackTraceElement ste = st[i];
+ sb.append("\tat ").append(ste.toString());
+ sb.append('\n');
+ }
+ if (i < st.length) {
+ sb.append("\t...");
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java
index 1bbd33831f..b063823b84 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.concurrent.locks.Lock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public interface IntentionLock {
+ Lock shareLock();
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java
new file mode 100644
index 0000000000..ea6677cd2d
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import static java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.kylin.common.KylinConfig;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+
+@Slf4j
+public class IntentionReadWriteLock implements IntentionLock, ReadWriteLock, MemoryLock {
+
+ private final InnerReentrantLock lockInternal = new InnerReentrantLock();
+ private final TransactionLock readLock;
+ private final TransactionLock writeLock;
+
+ private final IntentionShareLock intentionShareLock;
+
+ public IntentionReadWriteLock(IntentionShareLock intentionShareLock) {
+ this.intentionShareLock = intentionShareLock;
+ this.readLock = new InnerReadLock(lockInternal);
+ this.writeLock = new InnerWriteLock(lockInternal);
+ }
+
+ @Override
+ public TransactionLock readLock() {
+ return readLock;
+ }
+
+ @Override
+ public TransactionLock writeLock() {
+ return writeLock;
+ }
+
+ @Override
+ public Lock shareLock() {
+ return readLock;
+ }
+
+ protected long getLockTimeoutSeconds() {
+ return KylinConfig.getInstanceFromEnv().getLockTimeoutSeconds();
+ }
+
+ private static class InnerReentrantLock extends ReentrantReadWriteLock {
+ private final Set<Long> readLockOwners = new ConcurrentSkipListSet<>();
+
+ public InnerReentrantLock() {
+ super(true);
+ }
+
+ private Set<Long> getReadLockOwners() {
+ return this.readLockOwners;
+ }
+
+ private Set<Long> getReadLockDependencyThreads(long threadId) {
+ Thread writeLockOwner = getOwner();
+ Set<Long> dependencyThreads = new HashSet<>();
+ if (writeLockOwner != null) {
+ if (writeLockOwner.getId() != threadId) {
+ dependencyThreads.add(writeLockOwner.getId());
+ }
+ } else {
+ if (!getQueuedWriterThreads().isEmpty() && !readLockOwners.contains(threadId)) {
+ dependencyThreads.addAll(readLockOwners);
+ }
+ }
+ return dependencyThreads;
+ }
+
+ private Set<Long> getWriteLockDependencyThreads(long threadId) {
+ Set<Long> dependencyThreads = new HashSet<>();
+ Thread writeLockOwner = getOwner();
+ if (writeLockOwner != null) {
+ if (writeLockOwner.getId() != threadId) {
+ dependencyThreads.add(writeLockOwner.getId());
+ }
+ } else {
+ Preconditions.checkState(!readLockOwners.contains(threadId),
+ "Must release readLock before acquire writeLock");
+ dependencyThreads.addAll(readLockOwners);
+ }
+ return dependencyThreads;
+ }
+ }
+
+ public class InnerReadLock extends ReadLock implements TransactionLock {
+
+ /**
+ * Constructor for use by subclasses
+ *
+ * @param lock the outer lock object
+ * @throws NullPointerException if the lock is null
+ */
+ protected InnerReadLock(ReentrantReadWriteLock lock) {
+ super(lock);
+ }
+
+ @Override
+ public void lock() {
+ if (!isHeldByCurrentThread() && !this.tryLock()) {
+ throw new LockTimeoutException(this.transactionUnit());
+ }
+ lockInternal.getReadLockOwners().add(Thread.currentThread().getId());
+ }
+
+ @Override
+ public boolean tryLock() {
+ try {
+ intentionShareLock.lockIntentionLock();
+ return super.tryLock(IntentionReadWriteLock.this.getLockTimeoutSeconds(), TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ intentionShareLock.unlockIntentionLock();
+ log.warn("Failed to get lock: {}. This thread is interrupted.", transactionUnit());
+ throw new LockInterruptException(e);
+ }
+ }
+
+ @Override
+ public void unlock() {
+ if (isHeldByCurrentThread()) {
+ lockInternal.getReadLockOwners().remove(Thread.currentThread().getId());
+ super.unlock();
+ intentionShareLock.unlockIntentionLock();
+ }
+ }
+
+ @Override
+ public boolean isHeldByCurrentThread() {
+ return lockInternal.getReadHoldCount() > 0;
+ }
+
+ @Override
+ public Set<Long> getLockDependencyThreads(long threadId) {
+ Set<Long> dependencyThreads = new HashSet<>();
+ dependencyThreads.addAll(intentionShareLock.getReadLockDependencyThreads(threadId));
+ dependencyThreads.addAll(lockInternal.getReadLockDependencyThreads(threadId));
+ return dependencyThreads;
+ }
+
+ @Override
+ public TransactionLock getOppositeLock() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String transactionUnit() {
+ return IntentionReadWriteLock.this.transactionUnit();
+ }
+
+ @Override
+ public Long createTime() {
+ return IntentionReadWriteLock.this.createTime();
+ }
+
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return IntentionReadWriteLock.this.moduleEnum();
+ }
+ }
+
+ public class InnerWriteLock extends ReentrantReadWriteLock.WriteLock implements TransactionLock {
+
+ /**
+ * Constructor for use by subclasses
+ *
+ * @param lock the outer lock object
+ * @throws NullPointerException if the lock is null
+ */
+ protected InnerWriteLock(ReentrantReadWriteLock lock) {
+ super(lock);
+ }
+
+ @Override
+ public void lock() {
+ if (!isHeldByCurrentThread() && !this.tryLock()) {
+ throw new LockTimeoutException(this.transactionUnit());
+ }
+ }
+
+ @Override
+ public boolean tryLock() {
+ try {
+ intentionShareLock.lockIntentionLock();
+ return super.tryLock(IntentionReadWriteLock.this.getLockTimeoutSeconds(), TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ intentionShareLock.unlockIntentionLock();
+ log.warn("Failed to get lock: {}. This thread is interrupted.", transactionUnit());
+ throw new LockInterruptException(e);
+ }
+ }
+
+ @Override
+ public void unlock() {
+ super.unlock();
+ intentionShareLock.unlockIntentionLock();
+ }
+
+ @Override
+ public boolean isHeldByCurrentThread() {
+ return lockInternal.getWriteHoldCount() > 0;
+ }
+
+ @Override
+ public Set<Long> getLockDependencyThreads(long threadId) {
+ Set<Long> dependencyThreads = new HashSet<>(intentionShareLock.getReadLockDependencyThreads(threadId));
+ dependencyThreads.addAll(lockInternal.getWriteLockDependencyThreads(threadId));
+ return dependencyThreads;
+ }
+
+ @Override
+ public TransactionLock getOppositeLock() {
+ return readLock;
+ }
+
+ @Override
+ public String transactionUnit() {
+ return IntentionReadWriteLock.this.transactionUnit();
+ }
+
+ @Override
+ public Long createTime() {
+ return IntentionReadWriteLock.this.createTime();
+ }
+
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return IntentionReadWriteLock.this.moduleEnum();
+ }
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java
new file mode 100644
index 0000000000..50633690ce
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+
+public class IntentionShareLock implements IntentionLock {
+
+ private final Lock shareLock;
+
+ public IntentionShareLock() {
+ this.shareLock = null;
+ }
+
+ public IntentionShareLock(IntentionLock intentionLock) {
+ this.shareLock = intentionLock.shareLock();
+ }
+
+ @Override
+ public Lock shareLock() {
+ return shareLock;
+ }
+
+ public void lockIntentionLock() {
+ if (shareLock != null) {
+ shareLock.lock();
+ }
+ }
+
+ public void unlockIntentionLock() {
+ if (shareLock != null) {
+ shareLock.unlock();
+ }
+ }
+
+ public Set<Long> getReadLockDependencyThreads(Long threadId) {
+ if (shareLock instanceof IntentionReadWriteLock.InnerReadLock) {
+ return ((IntentionReadWriteLock.InnerReadLock) shareLock).getLockDependencyThreads(threadId);
+ }
+ return Collections.emptySet();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java
index 1bbd33831f..310ae29d7a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+public class LockInterruptException extends RuntimeException {
+ public LockInterruptException(Throwable throwable) {
+ super(throwable);
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java
similarity index 50%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java
index 86eb29c09f..180f85d9ac 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java
@@ -15,30 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.List;
+import java.util.Map;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+import javax.validation.constraints.NotNull;
-import com.fasterxml.jackson.annotation.JsonProperty;
+public interface LockManager {
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ ProjectLock getProjectLock(@NotNull String projectName);
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class QueryHistoryIdOffset extends RootPersistentEntity {
- @JsonProperty("query_history_id_offset")
- private long offset;
+ ProjectLock removeProjectLock(@NotNull String projectName);
- @JsonProperty("query_history_stat_id_offset")
- private long statMetaUpdateOffset;
+ ModuleLock getModuleLock(@NotNull String projectName, @NotNull ModuleLockEnum moduleName);
- public QueryHistoryIdOffset(long offset) {
- this.offset = offset;
- }
+ /** Since metadata is in json format, when we want to lock a path, we need lock multiple resources;
+ For example, if the pathName belongs to a model, we will lock model name、cc name、cc express. **/
+ List<PathLock> getPathLock(@NotNull String pathName);
+
+ String getProjectByPath(@NotNull String pathName);
+
+ Map<String, ProjectLock> listAllProjectLock();
+
+ void checkProjectLockSize();
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java
index 1bbd33831f..e03e335ecb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java
@@ -16,27 +16,20 @@
* limitations under the License.
*/
-package org.apache.kylin.metadata.favorite;
+package org.apache.kylin.common.persistence.lock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
-import com.fasterxml.jackson.annotation.JsonProperty;
+public class LockManagerInstance {
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ public static final LockManager INSTANCE;
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
+ static {
+ INSTANCE = (LockManager) ClassUtil.newInstance(KylinConfig.getInstanceFromEnv().getLockManager());
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ private LockManagerInstance() {
}
+
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java
index 1bbd33831f..b332f2ac88 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+public class LockTimeoutException extends RuntimeException {
+ public LockTimeoutException(String lockUnit) {
+ super(lockUnit);
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java
index 1bbd33831f..4eaf93db46 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java
@@ -15,28 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+public interface MemoryLock {
+ default String transactionUnit() {
+ return UnitOfWork.GLOBAL_UNIT;
+ }
- @JsonProperty("task_type")
- private String taskType;
+ default Long createTime() {
+ return 0L;
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ default ModuleLockEnum moduleEnum() {
+ return null;
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java
new file mode 100644
index 0000000000..b8d6c66815
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.guava30.shaded.common.collect.ComparisonChain;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * This graph only records nodes info, the edges are calculated at realtime.
+ */
+@Slf4j
+public class MemoryLockGraph {
+
+ private final Map<Long, ThreadNode> nodeMap = Maps.newConcurrentMap();
+
+ public void setThread(long threadId, TransactionLock requiredLock) {
+ ThreadNode node = nodeMap.computeIfAbsent(threadId, ThreadNode::new);
+ node.setRequiredLock(requiredLock);
+ }
+
+ public void deleteThreadNode(long threadId) {
+ nodeMap.remove(threadId);
+ }
+
+ public void resetThread(long threadId) {
+ nodeMap.get(threadId).onLocked();
+ }
+
+ public List<Long> preCheck(long threadId, Set<Long> dependencies) {
+ if (dependencies.isEmpty()) {
+ return Collections.emptyList();
+ }
+ ThreadNode target = nodeMap.get(threadId);
+ if (target == null) {
+ return Collections.emptyList();
+ }
+ List<ThreadNode> trace = new ArrayList<>();
+ for (Long id : dependencies) {
+ ThreadNode node = nodeMap.get(id);
+ if (node == null) {
+ continue;
+ }
+ if (findCycleForTarget(target, node, trace)) {
+ return trace.stream().map(ThreadNode::getThreadId).collect(Collectors.toList());
+ }
+ }
+ return Collections.emptyList();
+ }
+
+ private boolean findCycleForTarget(ThreadNode target, ThreadNode currentNode, List<ThreadNode> trace) {
+ if (currentNode.getThreadId() == target.getThreadId()) {
+ return true;
+ }
+ if (!trace.contains(currentNode)) {
+ trace.add(currentNode);
+ for (Long threadId : currentNode.getDependencies()) {
+ ThreadNode node = nodeMap.get(threadId);
+ if (node != null && findCycleForTarget(target, node, trace)) {
+ return true;
+ }
+ }
+ trace.remove(trace.size() - 1);
+ }
+ return false;
+ }
+
+ public List<List<Long>> checkForDeadLock() {
+ List<ThreadNode> trace = new ArrayList<>();
+ List<List<Long>> cycles = new ArrayList<>();
+ for (ThreadNode startNode : nodeMap.values()) {
+ Set<Long> finishedNode = cycles.stream().flatMap(List::stream).collect(Collectors.toSet());
+ findCycles(startNode, trace, cycles, finishedNode);
+ }
+ return cycles;
+ }
+
+ private void findCycles(ThreadNode currentNode, List<ThreadNode> trace, List<List<Long>> cycles,
+ Set<Long> finishedNode) {
+ if (finishedNode.contains(currentNode.getThreadId())) {
+ return;
+ }
+ int index = trace.indexOf(currentNode);
+ if (index != -1) {
+ cycles.add(trace.subList(index, trace.size()).stream().map(ThreadNode::getThreadId)
+ .collect(Collectors.toList()));
+ } else {
+ trace.add(currentNode);
+ currentNode.getDependencies().forEach(id -> {
+ ThreadNode node = nodeMap.get(id);
+ if (node != null) {
+ findCycles(node, trace, cycles, finishedNode);
+ }
+ });
+ trace.remove(trace.size() - 1);
+ }
+ }
+
+ public Set<Long> getKeyNodes(List<List<Long>> cycles) {
+ Map<Long, Integer> cycleCntMap = Maps.newHashMap();
+ cycles.stream().flatMap(List::stream).forEach(threadId -> {
+ int cnt = cycleCntMap.getOrDefault(threadId, 0);
+ cycleCntMap.put(threadId, cnt + 1);
+ });
+ Comparator<Long> comparator = ((Comparator<Long>) (thread1, thread2) -> ComparisonChain.start()
+ .compare(cycleCntMap.get(thread1), cycleCntMap.get(thread2))
+ .compare(nodeMap.get(thread1).getCreateTime(), nodeMap.get(thread2).getCreateTime()).result())
+ .reversed();
+ Set<Long> keyNode = new HashSet<>();
+ cycles.forEach(cycle -> {
+ cycle.sort(comparator);
+ keyNode.add(cycle.get(0));
+ });
+ return keyNode;
+ }
+
+ public boolean isThreadWaitForLock(long threadId) {
+ return nodeMap.get(threadId).getRequiredLock() == null;
+ }
+
+ public int getThreadNodeCount() {
+ return nodeMap.size();
+ }
+
+ @Data
+ static class ThreadNode {
+ long threadId;
+ long createTime;
+ TransactionLock requiredLock;
+
+ private ThreadNode(long threadId) {
+ this.threadId = threadId;
+ this.createTime = System.currentTimeMillis();
+ }
+
+ private void onLocked() {
+ this.requiredLock = null;
+ }
+
+ private Set<Long> getDependencies() {
+ return requiredLock == null ? Collections.emptySet() : requiredLock.getLockDependencyThreads(threadId);
+ }
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java
new file mode 100644
index 0000000000..e6f816156a
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.validation.constraints.NotNull;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+
+import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MemoryLockManager implements LockManager {
+ private final Map<String, ProjectLock> projectLockMap = Maps.newConcurrentMap();
+ private final Map<String, ModuleLock> moduleLockMap = Maps.newConcurrentMap();
+ private final Map<String, PathLock> pathLockMap = Maps.newConcurrentMap();
+
+ // workaround for https://bugs.openjdk.org/browse/JDK-8161372
+ private <T> T computeIfAbsent(Map<String, T> map, String key, Function<String, ? extends T> mappingFunction) {
+ T result = map.get(key);
+ if (null == result) {
+ result = map.computeIfAbsent(key, mappingFunction);
+ }
+ return result;
+ }
+
+ @Override
+ public ProjectLock getProjectLock(@NotNull String projectName) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(projectName), "projectName is blank");
+ return computeIfAbsent(projectLockMap, projectName, k -> new ProjectLock(projectName));
+ }
+
+ @Override
+ public ProjectLock removeProjectLock(@NotNull String projectName) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(projectName), "projectName is blank");
+ return projectLockMap.remove(projectName);
+ }
+
+ @Override
+ public ModuleLock getModuleLock(@NotNull String projectName, @NotNull ModuleLockEnum moduleName) {
+ Preconditions.checkNotNull(moduleName, "moduleName is null");
+ return computeIfAbsent(moduleLockMap, projectName + ":" + moduleName,
+ k -> new ModuleLock(moduleName, getProjectLock(projectName)));
+ }
+
+ @Override
+ public List<PathLock> getPathLock(@NotNull String pathName) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(pathName), "pathName is blank");
+ // parse the Module in the Path, and return the DEFAULT Module if it cannot be parsed
+ val resourcePath = ResourcePathParser.parseResourcePath(pathName);
+ ModuleLockEnum moduleLockEnum = ModuleLockEnum.getModuleEnum(resourcePath.getModule());
+
+ ModuleLock moduleLock = getModuleLock(resourcePath.getProject(), moduleLockEnum);
+ List<String> newPaths = ResourcePathParser.transformPath(pathName, moduleLockEnum, resourcePath);
+ return newPaths.stream().map(path -> computeIfAbsent(pathLockMap, path, k -> new PathLock(path, moduleLock)))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public String getProjectByPath(@NotNull String pathName) {
+ return ResourcePathParser.parseResourcePath(pathName).getProject();
+ }
+
+ @Override
+ public Map<String, ProjectLock> listAllProjectLock() {
+ return Maps.newHashMap(projectLockMap);
+ }
+
+ @Override
+ public void checkProjectLockSize() {
+ if (projectLockMap.size() > 5000L) {
+ projectLockMap.forEach((k, v) -> log.warn("lock leaks lock: {}，num: {}", k, v.getTranHoldCount()));
+ }
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java
new file mode 100644
index 0000000000..32c610427f
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import static org.apache.kylin.common.persistence.transaction.TransactionManagerInstance.INSTANCE;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.RawResource;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.persistence.ThreadViewResourceStore;
+import org.apache.kylin.common.persistence.metadata.JdbcMetadataStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MemoryLockUtils {
+
+ private MemoryLockUtils() {
+ }
+
+ private static final MemoryLockGraph MEMORY_LOCK_GRAPH = new MemoryLockGraph();
+
+ public static List<TransactionLock> getProjectLock(String project) {
+ return Collections.singletonList(INSTANCE.getProjectLock(project));
+ }
+
+ public static List<TransactionLock> getModuleLocks(String project, ModuleLockEnum module) {
+ return getModuleLocks(project, Collections.singletonList(module));
+ }
+
+ public static List<TransactionLock> getModuleLocks(String project, Collection<ModuleLockEnum> modules) {
+ return modules.stream().filter(Objects::nonNull).map(module -> INSTANCE.getModuleLock(project, module))
+ .distinct().collect(Collectors.toList());
+ }
+
+ public static List<TransactionLock> getPathLocks(String path) {
+ return getPathLocks(path, false);
+ }
+
+ public static List<TransactionLock> getPathLocks(String path, boolean readOnly) {
+ return getPathLocks(Collections.singletonList(path), readOnly);
+ }
+
+ public static List<TransactionLock> getPathLocks(Collection<String> paths) {
+ return getPathLocks(paths, false);
+ }
+
+ public static List<TransactionLock> getPathLocks(Collection<String> paths, boolean readOnly) {
+ return paths.stream().filter(StringUtils::isNotBlank).map(path -> INSTANCE.getPathLock(path, readOnly))
+ .flatMap(List::stream).distinct().collect(Collectors.toList());
+ }
+
+ public static <T extends RootPersistentEntity> T doWithLock(T entity, String resPath, boolean readonly,
+ ResourceStore store, Action<T> action) {
+ if (isNoNeedToLock(readonly, store)) {
+ return action.run();
+ }
+ if (entity == null) {
+ MemoryLockUtils.lockAndRecord(resPath, null, readonly);
+ return action.run();
+ }
+
+ List<String> lockPaths = readonly? Collections.singletonList(resPath) : entity.getLockPaths(resPath);
+ MemoryLockUtils.lockAndRecord(resPath, lockPaths, readonly);
+
+ T newEntity = action.run();
+
+ if (!readonly && newEntity != null) {
+ // update locks with new entity
+ List<String> newLockPaths = newEntity.getLockPaths(resPath);
+ List<String> delta = newLockPaths.stream().filter(path -> !lockPaths.contains(path))
+ .collect(Collectors.toList());
+ if (!delta.isEmpty()) {
+ MemoryLockUtils.lockAndRecord(resPath, delta, false);
+ }
+ }
+ return newEntity;
+ }
+
+ public static boolean isNoNeedToLock(boolean readonly, ResourceStore store) {
+ return !UnitOfWork.isAlreadyInTransaction()
+ || !readonly && !(store instanceof ThreadViewResourceStore)
+ || !(store.getMetadataStore() instanceof JdbcMetadataStore)
+ && !KylinConfig.getInstanceFromEnv().isUTEnv()
+ || UnitOfWork.get().getParams().isUseProjectLock();
+ }
+
+ public static RawResource doWithLock(String resPath, boolean readonly, ResourceStore store,
+ Action<RawResource> action) {
+ if (!isNoNeedToLock(readonly, store)) {
+ lockAndRecord(resPath, null, readonly);
+ }
+ return action.run();
+ }
+
+ public static void manuallyLockModule(String project, ModuleLockEnum module, ResourceStore store) {
+ if (isNoNeedToLock(false, store)) {
+ return;
+ }
+ List<TransactionLock> moduleLocks = getModuleLocks(project, module);
+ assert moduleLocks.size() == 1;
+ batchLock(moduleLocks.get(0).transactionUnit(), moduleLocks, false);
+ }
+
+ public static void lockAndRecord(String resPath, List<String> specifiedPath, boolean readonly){
+
+ if (resPath.equals("/")) {
+ log.warn("The resPath of Entity is wrong. Cannot lock this path.", new Throwable());
+ return;
+ }
+
+ List<String> paths = specifiedPath == null ? Collections.singletonList(resPath) : specifiedPath;
+
+ if (readonly) {
+ Set<String> holdReadLocks = UnitOfWork.get().getReadLockPath();
+ paths = paths.stream().filter(path -> !holdReadLocks.contains(path)).collect(Collectors.toList());
+ }
+
+ if (paths.isEmpty()) {
+ return;
+ }
+ List<TransactionLock> locks = getPathLocks(paths, readonly);
+
+ batchLock(resPath, locks, readonly);
+ }
+
+ private static void batchLock(String resPath, List<TransactionLock> locks, boolean readonly) {
+ for (TransactionLock lock : locks) {
+ if (lock instanceof IntentionReadWriteLock.InnerWriteLock) {
+ TransactionLock readLock = lock.getOppositeLock();
+ readLock.unlock();
+ }
+ }
+
+ locks.forEach(lock -> {
+ lockWithPreCheck(lock);
+ UnitOfWork.recordLocks(resPath, lock, readonly);
+ });
+ }
+
+ private static void lockWithPreCheck(TransactionLock lock) {
+ if (lock instanceof IntentionReadWriteLock.InnerWriteLock) {
+ TransactionLock readLock = lock.getOppositeLock();
+ readLock.unlock();
+ }
+ long threadID = Thread.currentThread().getId();
+ Set<Long> dependencyThreads = lock.getLockDependencyThreads(threadID);
+ List<Long> cycle = MEMORY_LOCK_GRAPH.preCheck(threadID, dependencyThreads);
+
+ if (cycle.isEmpty()) {
+ MEMORY_LOCK_GRAPH.setThread(threadID, lock);
+ lock.lock();
+ MEMORY_LOCK_GRAPH.resetThread(threadID);
+ } else {
+ StringBuilder builder = new StringBuilder();
+ Thread.getAllStackTraces().keySet().stream().filter(t -> cycle.contains(t.getId()))
+ .forEach(thread -> builder.append(Arrays.toString(thread.getStackTrace())));
+ throw new DeadLockException(
+ "Lock preCheck failed! To avoid deadlock, restart this transaction." + builder.toString());
+ }
+ }
+
+ public static Set<Long> findDeadLockThreadIds() {
+ List<List<Long>> cycles = MEMORY_LOCK_GRAPH.checkForDeadLock();
+ return MEMORY_LOCK_GRAPH.getKeyNodes(cycles);
+ }
+
+ public static void removeThreadFromGraph() {
+ MEMORY_LOCK_GRAPH.deleteThreadNode(Thread.currentThread().getId());
+ }
+
+ public static int getTransactionSize() {
+ return MEMORY_LOCK_GRAPH.getThreadNodeCount();
+ }
+
+ public interface Action<T> {
+ T run();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java
similarity index 51%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java
index dc22af1698..ff7629335a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java
@@ -15,33 +15,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.metadata.usergroup;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
+package org.apache.kylin.common.persistence.lock;
import lombok.Getter;
-import lombok.Setter;
-@SuppressWarnings("serial")
@Getter
-@Setter
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class UserGroup extends RootPersistentEntity {
- @JsonProperty("group_name")
- private String groupName;
+public class ModuleLock extends AbstractMemoryLock {
+
+ private final ModuleLockEnum moduleName;
+ private final ProjectLock projectLock;
+
+ public ModuleLock(ModuleLockEnum moduleName, ProjectLock projectLock) {
+ super(projectLock);
+ this.moduleName = moduleName;
+ this.projectLock = projectLock;
+ }
- public UserGroup() {
+ @Override
+ public String transactionUnit() {
+ return projectLock.transactionUnit() + ":" + moduleName;
}
- public UserGroup(String groupName) {
- this.groupName = groupName;
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return moduleName;
+ }
+
+ @Override
+ public TransactionLock readLock() {
+ throw new UnsupportedOperationException("module don's support read lock!");
}
@Override
- public String resourceName() {
- return groupName;
+ protected long getLockTimeoutSeconds() {
+ return super.getLockTimeoutSeconds() * 3;
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java
new file mode 100644
index 0000000000..9caa06ab53
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_EXPRESSION;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_NAME;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.DATAFLOW;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.DATAFLOW_DETAILS;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.EXECUTE;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.INDEX_PLAN;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.MODEL_DESC;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.TABLE_EXD;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+
+public enum ModuleLockEnum {
+ JOB(EXECUTE.value()), //
+ MODEL(MODEL_DESC.value(), DATAFLOW.value(), DATAFLOW_DETAILS.value(), INDEX_PLAN.value()), //
+ TABLE(ResourceEnum.TABLE.value(), TABLE_EXD.value()), //
+ HISTORY_SOURCE_USAGE(ResourceEnum.HISTORY_SOURCE_USAGE.value()), //
+ PROJECT(ResourceEnum.PROJECT.value()), //
+ COMPUTE_COLUMN(COMPUTE_COLUMN_NAME.value(), COMPUTE_COLUMN_EXPRESSION.value()), //
+ DEFAULT();
+
+ private final List<String> moduleNames;
+
+ public List<String> getModuleNames() {
+ return moduleNames;
+ }
+
+ ModuleLockEnum(String... moduleNames) {
+ this.moduleNames = Lists.newArrayList(moduleNames);
+ }
+
+ private static final Map<String, ModuleLockEnum> MODULE_LOCK_ENUMS = Maps.newHashMap();
+
+ static {
+ Arrays.stream(values()).forEach(moduleLockEnum -> moduleLockEnum.moduleNames
+ .forEach(module -> MODULE_LOCK_ENUMS.put(module, moduleLockEnum)));
+ }
+
+ public static ModuleLockEnum getModuleEnum(String moduleName) {
+ if (StringUtils.isBlank(moduleName)) {
+ return DEFAULT;
+ }
+ return MODULE_LOCK_ENUMS.getOrDefault(moduleName, DEFAULT);
+ }
+
+ public enum ResourceEnum {
+ EXECUTE,
+ MODEL_DESC,
+ DATAFLOW,
+ DATAFLOW_DETAILS,
+ INDEX_PLAN,
+ TABLE,
+ TABLE_EXD,
+ HISTORY_SOURCE_USAGE,
+ PROJECT,
+ COMPUTE_COLUMN_NAME,
+ COMPUTE_COLUMN_EXPRESSION;
+
+ public String value() {
+ return name().toLowerCase();
+ }
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java
index 1bbd33831f..c6388cf2cb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java
@@ -15,28 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public class OptimisticLockRetryException extends RuntimeException {
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java
index 1bbd33831f..15d7120b30 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java
@@ -15,28 +15,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+public class PathLock extends AbstractMemoryLock implements TempLock {
+
+ private final String pathName;
+ private final ModuleLock moduleLock;
- @JsonProperty("task_type")
- private String taskType;
+ public PathLock(String pathName, ModuleLock moduleLock) {
+ super(moduleLock);
+ this.pathName = pathName;
+ this.moduleLock = moduleLock;
+ }
+
+ @Override
+ public String transactionUnit() {
+ return moduleLock.transactionUnit() + ":" + pathName;
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return moduleLock.getModuleName();
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java
similarity index 58%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java
index 1bbd33831f..d928abbf68 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java
@@ -15,28 +15,32 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import lombok.Getter;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+@Getter
+public class ProjectLock extends AbstractMemoryLock {
-import com.fasterxml.jackson.annotation.JsonProperty;
+ private final String projectName;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ public ProjectLock(String projectUnit) {
+ super(new IntentionShareLock());
+ this.projectName = projectUnit;
+ }
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+ @Override
+ public String transactionUnit() {
+ return projectName;
+ }
- @JsonProperty("task_type")
- private String taskType;
+ @Override
+ public TransactionLock readLock() {
+ throw new UnsupportedOperationException("project don's support read lock!");
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ @Override
+ protected long getLockTimeoutSeconds() {
+ return super.getLockTimeoutSeconds() * 10;
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java
new file mode 100644
index 0000000000..14ecfa021b
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.DEFAULT;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.base.Joiner;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.cache.Cache;
+import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.glassfish.jersey.uri.UriTemplate;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+@UtilityClass
+@Slf4j
+public class ResourcePathParser {
+
+ private static final int CACHE_SIZE = 10000;
+ private static final int CACHE_EXPIRE_MINUTES = 30;
+ private static final String RESOURCE_PATH_SUFFIX = ".json";
+ private static final Cache<String, ResourcePath> RES_CACHE = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE)
+ .expireAfterAccess(CACHE_EXPIRE_MINUTES, TimeUnit.MINUTES).build();
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final List<String> TEMPLATE_URIS;
+ private static final List<UriTemplate> TEMPLATES;
+ public static final Joiner JOINER = Joiner.on('/').skipNulls();
+
+ static {
+ TEMPLATE_URIS = Lists.newArrayList(//
+ "/{project}/{module}/{resource1}", // normal
+ "/{project}/{module}/{resource1}/{resource2}", // dataflow_detail
+ "/{project}/{module}", // _health
+ "/{project}" // UUID
+ );
+ TEMPLATES = TEMPLATE_URIS.stream().map(UriTemplate::new).collect(Collectors.toList());
+ }
+
+ public static ResourcePath parseResourcePath(String originPath) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(originPath), "path is blank");
+ if (originPath.endsWith(RESOURCE_PATH_SUFFIX)) {
+ originPath = originPath.substring(0, originPath.length() - RESOURCE_PATH_SUFFIX.length());
+ }
+
+ ResourcePath resourcePath = RES_CACHE.getIfPresent(originPath);
+ if (Objects.nonNull(resourcePath)) {
+ return resourcePath;
+ }
+
+ for (UriTemplate template : TEMPLATES) {
+ Map<String, String> map = Maps.newHashMap();
+ if (template.match(originPath, map)) {
+ resourcePath = MAPPER.convertValue(map, ResourcePath.class);
+ RES_CACHE.put(originPath, resourcePath);
+ return resourcePath;
+ }
+ }
+ throw new IllegalArgumentException("The path does not match: " + originPath);
+ }
+
+ public static List<String> transformPath(String originPath, ModuleLockEnum moduleLockEnum,
+ ResourcePathParser.ResourcePath resourcePath) {
+ if (moduleLockEnum.equals(DEFAULT) || Objects.isNull(resourcePath)) {
+ // can not parse module
+ return Collections.singletonList(originPath);
+ }
+
+ // transform path name
+ return resourcePath.transformPath();
+ }
+
+ @AllArgsConstructor
+ @NoArgsConstructor
+ @Data
+ public static class ResourcePath {
+ private String project;
+ private String module;
+ private String resource1;
+ private String resource2;
+
+ public List<String> transformPath() {
+ return getDefaultLockPath();
+ }
+
+ private List<String> getDefaultLockPath() {
+ return Collections.singletonList(JOINER.join(project, module, resource1, resource2));
+ }
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java
index 1bbd33831f..8803befff8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java
@@ -15,28 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public interface TempLock {
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java
new file mode 100644
index 0000000000..275c337fc2
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.NamedThreadFactory;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.guava30.shaded.common.primitives.Longs;
+
+@Slf4j
+public class TransactionDeadLockHandler implements DeadLockHandler, Runnable {
+
+ public static final String THREAD_NAME_PREFIX = "Transaction-Thread";
+ private final ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+ private final AtomicBoolean scheduleStarted = new AtomicBoolean(false);
+ private final long interval = KylinConfig.getInstanceFromEnv().getLockCheckIntervalSeconds();
+ private final ScheduledExecutorService checkerPool = Executors.newScheduledThreadPool(1,
+ new NamedThreadFactory("DeadLockChecker"));
+
+ private int checkCnt = 0;
+ private double checkAvgCost = 0;
+ private long checkMaxCost = 0;
+
+ private TransactionDeadLockHandler() {
+ }
+
+ private static final class InstanceHolder {
+ static final TransactionDeadLockHandler INSTANCE = new TransactionDeadLockHandler();
+ }
+
+ public static TransactionDeadLockHandler getInstance() {
+ return InstanceHolder.INSTANCE;
+ }
+
+ public void start() {
+ if (scheduleStarted.compareAndSet(false, true)) {
+ log.info("Start scheduling deadlock detection every {} seconds", interval);
+ checkerPool.scheduleWithFixedDelay(this, 1, interval, TimeUnit.SECONDS);
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ Set<ThreadInfo> threadInfoSet = getThreadsToBeKill();
+ if (CollectionUtils.isEmpty(threadInfoSet)) {
+ return;
+ }
+
+ for (ThreadInfo threadInfo : threadInfoSet) {
+ log.warn("Deadlock thread detail:\n{}", threadInfo);
+ }
+ Set<Long> threadIdSet = threadInfoSet.stream().map(ThreadInfo::getThreadId).collect(Collectors.toSet());
+ killThreadsById(threadIdSet);
+ } catch (Throwable throwable) {
+ log.info("Deadlock detection failed. Wait for next execution ...", throwable);
+ }
+ }
+
+ @Override
+ public Set<ThreadInfo> getThreadsToBeKill() {
+ ThreadInfo[] threadInfos = findDeadLockThreads();
+ if (ArrayUtils.isEmpty(threadInfos)) {
+ return Collections.emptySet();
+ }
+ Set<ThreadInfo> threadInfoSet = Arrays.stream(threadInfos)//
+ .filter(Objects::nonNull)//
+ .filter(t -> StringUtils.startsWith(t.getThreadName(), getThreadNamePrefix()))
+ .collect(Collectors.toSet());
+ if (CollectionUtils.isEmpty(threadInfoSet)) {
+ return Collections.emptySet();
+ }
+ log.info("Found transaction deadlock thread, size {}, total transaction size: {}", threadInfoSet.size(),
+ MemoryLockUtils.getTransactionSize());
+ return threadInfoSet;
+ }
+
+ @Override
+ public Set<Long> getThreadIdToBeKill() {
+ return getThreadsToBeKill().stream().filter(Objects::nonNull).map(ThreadInfo::getThreadId)
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public ThreadInfo[] findDeadLockThreads() {
+ long start = System.currentTimeMillis();
+ long[] threadIds = Longs.toArray(MemoryLockUtils.findDeadLockThreadIds());
+ updateTimeStatistics(System.currentTimeMillis() - start);
+ if (ArrayUtils.isEmpty(threadIds)) {
+ return new ThreadInfo[0];
+ }
+ return mxBean.getThreadInfo(threadIds, true, true);
+ }
+
+ private void updateTimeStatistics(long timeCost) {
+ checkAvgCost = (checkAvgCost * checkCnt + timeCost) / (checkCnt + 1);
+ checkCnt = checkCnt + 1;
+ checkMaxCost = Math.max(checkMaxCost, timeCost);
+ if (checkCnt % 200 == 0) {
+ log.debug(
+ "DeadLock checker has run {} times, the average cost time is {} ms, and the max cost time is {} ms.",
+ checkCnt, checkAvgCost, checkMaxCost);
+ }
+ }
+
+ @Override
+ public Set<Thread> getThreadsById(Set<Long> threadIdSet) {
+ if (CollectionUtils.isEmpty(threadIdSet)) {
+ return Collections.emptySet();
+ }
+ return Thread.getAllStackTraces().keySet().stream().filter(t -> threadIdSet.contains(t.getId()))
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public void killThreads(Set<Thread> threads) {
+ for (Thread thread : threads) {
+ log.info("Interrupt thread: {}", thread.getName());
+ thread.interrupt();
+ }
+ }
+
+ @Override
+ public void killThreadsById(Set<Long> threadIdSet) {
+ if (CollectionUtils.isEmpty(threadIdSet)) {
+ return;
+ }
+ killThreads(getThreadsById(threadIdSet));
+ }
+
+ @Override
+ public String getThreadNamePrefix() {
+ return THREAD_NAME_PREFIX;
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java
index 1bbd33831f..115228c940 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java
@@ -15,28 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public interface TransactionLock extends Lock, MemoryLock {
+ boolean isHeldByCurrentThread();
+ Set<Long> getLockDependencyThreads(long threadId);
+ TransactionLock getOppositeLock();
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java
similarity index 51%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java
index 86eb29c09f..fdade6e10b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java
@@ -15,30 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.List;
+import java.util.Map;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public interface TransactionLockManager {
-import com.fasterxml.jackson.annotation.JsonProperty;
+ TransactionLock getLock(String project, boolean readonly);
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ TransactionLock getTempLock(String project, boolean readonly);
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class QueryHistoryIdOffset extends RootPersistentEntity {
- @JsonProperty("query_history_id_offset")
- private long offset;
+ void removeLock(String project);
- @JsonProperty("query_history_stat_id_offset")
- private long statMetaUpdateOffset;
+ Map<String, ProjectLock> getProjectLocksForRead();
- public QueryHistoryIdOffset(long offset) {
- this.offset = offset;
- }
+ TransactionLock getProjectLock(String projectName);
+
+ TransactionLock getModuleLock(String projectName, ModuleLockEnum moduleName);
+
+ /** Since metadata is in json format, when we want to lock a path, we need lock multiple resources;
+ For example, if the pathName belongs to a model, we will lock model name、cc name、cc express. **/
+ List<TransactionLock> getPathLock(String pathName, boolean readOnly);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java
index 1bbd33831f..6bc98e9d20 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java
@@ -15,28 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import lombok.experimental.Delegate;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionProjectLock implements TransactionLock {
-import com.fasterxml.jackson.annotation.JsonProperty;
+ @Delegate
+ private TransactionLock lock;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public TransactionProjectLock(TransactionLock lock) {
+ this.lock = lock;
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java
index 1bbd33831f..ca9a14a117 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java
@@ -15,28 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import lombok.experimental.Delegate;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionTempLock implements TransactionLock, TempLock {
+ @Delegate
+ private TransactionLock lock;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public TransactionTempLock(TransactionLock lock) {
+ this.lock = lock;
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java
index 1bbd33831f..d41ccabf63 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java
@@ -15,28 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock.rule;
-package org.apache.kylin.metadata.favorite;
+public abstract class LockEliminationRule {
+ private LockEliminationRule next;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
+ public void setNext(LockEliminationRule next) {
+ this.next = next;
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public void handle(LockInfo lockInfo) {
+ this.doHandle(lockInfo);
+ if (next != null) {
+ next.handle(lockInfo);
+ }
}
+
+ protected abstract void doHandle(LockInfo lockInfo);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java
new file mode 100644
index 0000000000..d2bc22bf99
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+
+public class LockEliminationRuleExec {
+ private final LockEliminationRule ruleChain;
+
+ private LockEliminationRuleExec() {
+ // projectRule -> moduleRule -> pathRule
+ ProjectEliminationRule projectRule = new ProjectEliminationRule();
+ ModuleEliminationRule moduleRule = new ModuleEliminationRule();
+ PathEliminationRule pathRule = new PathEliminationRule();
+ moduleRule.setNext(pathRule);
+ projectRule.setNext(moduleRule);
+ ruleChain = projectRule;
+ }
+
+ private static final class InstanceHolder {
+ static final LockEliminationRuleExec INSTANCE = new LockEliminationRuleExec();
+ }
+
+ public static LockEliminationRuleExec getInstance() {
+ return InstanceHolder.INSTANCE;
+ }
+
+ /**
+ * Execute the lock elimination rule chain.
+ */
+ public LockInfo execute(String project, List<TransactionLock> projectLocks, List<TransactionLock> moduleLocks,
+ List<TransactionLock> pathLocks) {
+ LockInfo lockInfo = new LockInfo(project, projectLocks, moduleLocks, pathLocks);
+ if (projectLocks.isEmpty() && moduleLocks.isEmpty() && pathLocks.isEmpty()) {
+ return lockInfo;
+ }
+ ruleChain.handle(lockInfo);
+ return lockInfo;
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java
new file mode 100644
index 0000000000..426f6b2b04
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+
+import lombok.Data;
+
+@Data
+public class LockInfo {
+ private String project;
+ private List<TransactionLock> projectLocks;
+ private List<TransactionLock> moduleLocks;
+ private List<TransactionLock> pathLocks;
+
+ public LockInfo(String project, List<TransactionLock> projectLocks, List<TransactionLock> moduleLocks,
+ List<TransactionLock> pathLocks) {
+ this.project = project;
+ this.projectLocks = projectLocks;
+ this.moduleLocks = moduleLocks;
+ this.pathLocks = pathLocks;
+ }
+
+ public List<TransactionLock> getAllLocks() {
+ List<TransactionLock> result = Lists.newArrayList();
+ result.addAll(projectLocks);
+ result.addAll(moduleLocks);
+ result.addAll(pathLocks);
+ return result;
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java
new file mode 100644
index 0000000000..34df00cc27
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * If the path locks are included by module locks, the path locks will be cleared.
+ */
+@Slf4j
+public class ModuleEliminationRule extends LockEliminationRule {
+ @Override
+ protected void doHandle(LockInfo lockInfo) {
+ if (lockInfo.getPathLocks().isEmpty() || lockInfo.getModuleLocks().isEmpty()) {
+ return;
+ }
+
+ List<ModuleLockEnum> modules = lockInfo.getModuleLocks().stream().map(TransactionLock::moduleEnum)
+ .collect(Collectors.toList());
+ Map<ModuleLockEnum, List<TransactionLock>> modulePathMap = lockInfo.getPathLocks().stream()
+ .collect(Collectors.groupingBy(TransactionLock::moduleEnum));
+ modulePathMap.forEach((module, pathLocks) -> {
+ if (modules.contains(module)) {
+ pathLocks.clear();
+ log.info("path lock has been included by module lock [{}]", module);
+ }
+ });
+ List<TransactionLock> newPathLocks = modulePathMap.values().stream().filter(CollectionUtils::isNotEmpty)
+ .flatMap(List::stream).collect(Collectors.toList());
+ lockInfo.setPathLocks(newPathLocks);
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java
new file mode 100644
index 0000000000..4f878839cb
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * If the number of path locks exceeds the maximum size
+ * the path locks will be cleared and the corresponding module locks will be added.
+ */
+@Slf4j
+public class PathEliminationRule extends LockEliminationRule {
+
+ @Override
+ protected void doHandle(LockInfo lockInfo) {
+ if (lockInfo.getPathLocks().isEmpty()) {
+ return;
+ }
+
+ int maxSize = KylinConfig.getInstanceFromEnv().getLockRemoveMaxSize();
+ Map<ModuleLockEnum, List<TransactionLock>> modulePathMap = lockInfo.getPathLocks().stream()
+ .collect(Collectors.groupingBy(TransactionLock::moduleEnum));
+
+ List<ModuleLockEnum> moduleNames = Lists.newArrayList();
+ modulePathMap.forEach((module, pathLocks) -> {
+ if (pathLocks.size() > maxSize) {
+ pathLocks.clear();
+ moduleNames.add(module);
+ }
+ });
+
+ if (moduleNames.isEmpty()) {
+ return;
+ }
+ log.info("add new module locks: [{}]", StringUtils.join(moduleNames, ","));
+ List<TransactionLock> newPathLocks = modulePathMap.values().stream().filter(CollectionUtils::isNotEmpty)
+ .flatMap(List::stream).collect(Collectors.toList());
+ List<TransactionLock> newModuleLocks = MemoryLockUtils.getModuleLocks(lockInfo.getProject(), moduleNames);
+
+ lockInfo.setPathLocks(newPathLocks);
+ lockInfo.getModuleLocks().addAll(newModuleLocks);
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java
similarity index 58%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java
index 1bbd33831f..2e0ca80c39 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java
@@ -15,28 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock.rule;
-package org.apache.kylin.metadata.favorite;
+import lombok.extern.slf4j.Slf4j;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+/**
+ * If the project locks is empty, then the module locks and path locks will be cleared.
+ */
+@Slf4j
+public class ProjectEliminationRule extends LockEliminationRule {
+ @Override
+ protected void doHandle(LockInfo lockInfo) {
+ if (lockInfo.getProjectLocks().isEmpty()) {
+ return;
+ }
+ lockInfo.getModuleLocks().clear();
+ lockInfo.getPathLocks().clear();
+ log.info("the project locks is not empty, clean up the module locks and path locks");
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
index 4a119bf3e5..d8c5c2e33d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
@@ -99,6 +99,7 @@ public class JdbcMetadataStore extends MetadataStore {
@Getter
private final JdbcTemplate jdbcTemplate;
private final String table;
+ private final boolean isUT;
public JdbcMetadataStore(KylinConfig config) throws Exception {
super(config);
@@ -108,6 +109,7 @@ public class JdbcMetadataStore extends MetadataStore {
transactionManager = new DataSourceTransactionManager(dataSource);
jdbcTemplate = new JdbcTemplate(dataSource);
table = url.getIdentifier();
+ isUT = config.isUTEnv();
if (config.isMetadataAuditLogEnabled()) {
auditLogStore = new JdbcAuditLogStore(config, jdbcTemplate, transactionManager,
table + JdbcAuditLogStore.AUDIT_LOG_SUFFIX);
@@ -129,6 +131,7 @@ public class JdbcMetadataStore extends MetadataStore {
String.format(Locale.ROOT, SELECT_BY_KEY_MVCC_SQL, table, path, mvcc - 1),
RAW_RESOURCE_ROW_MAPPER);
if (CollectionUtils.isEmpty(result)) {
+ assert isUT || mvcc == 0;
affectedRow = insert(String.format(Locale.ROOT, INSERT_SQL, table), path, bs, ts, mvcc);
} else {
affectedRow = update(String.format(Locale.ROOT, UPDATE_SQL, table), bs, mvcc, ts, path,
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java
index 1fc9c6a7a6..f802d32e95 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java
@@ -89,6 +89,7 @@ public abstract class MetadataStore {
public void batchUpdate(UnitMessages unitMessages, boolean skipAuditLog, String unitPath, long epochId)
throws Exception {
+ UnitOfWork.get().onStartUnitUpdate();
for (Event event : unitMessages.getMessages()) {
if (event instanceof ResourceCreateOrUpdateEvent) {
val rawResource = ((ResourceCreateOrUpdateEvent) event).getCreatedOrUpdated();
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java
index 263e020cff..3877a191da 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java
@@ -27,7 +27,6 @@ import org.apache.kylin.common.persistence.UnitMessages;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
import org.apache.kylin.common.scheduler.EventBusFactory;
-
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Setter;
@@ -115,7 +114,7 @@ public class MessageSynchronization {
}
public void replayAllMetadata(boolean needCloseReplay) throws IOException {
- val lockKeys = Lists.newArrayList(TransactionLock.getProjectLocksForRead().keySet());
+ val lockKeys = Lists.newArrayList(TransactionManagerInstance.INSTANCE.getProjectLocksForRead().keySet());
lockKeys.sort(Comparator.naturalOrder());
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
try {
@@ -124,7 +123,7 @@ public class MessageSynchronization {
resourceStore.getAuditLogStore().pause();
}
for (String lockKey : lockKeys) {
- TransactionLock.getLock(lockKey, false).lock();
+ TransactionManagerInstance.INSTANCE.getProjectLock(lockKey).lock();
}
log.info("Acquired all locks, start to reload");
@@ -133,7 +132,7 @@ public class MessageSynchronization {
} finally {
Collections.reverse(lockKeys);
for (String lockKey : lockKeys) {
- TransactionLock.getLock(lockKey, false).unlock();
+ TransactionManagerInstance.INSTANCE.getProjectLock(lockKey).unlock();
}
if (needCloseReplay) {
// if not stop, reinit return directly
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java
deleted file mode 100644
index 6807331cf2..0000000000
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.persistence.transaction;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import lombok.experimental.Delegate;
-
-public class TransactionLock {
-
- static Map<String, Pair<ReentrantReadWriteLock, AtomicLong>> projectLocks = Maps.newConcurrentMap();
- private static final Logger logger = LoggerFactory.getLogger(TransactionLock.class);
-
- public static TransactionLock getLock(String project, boolean readonly) {
- Pair<ReentrantReadWriteLock, AtomicLong> lockPair = projectLocks.get(project);
- if (lockPair == null) {
- synchronized (UnitOfWork.class) {
- lockPair = projectLocks.computeIfAbsent(project,
- k -> Pair.newPair(new ReentrantReadWriteLock(true), new AtomicLong()));
- }
- }
- lockPair.getSecond().incrementAndGet();
- ReentrantReadWriteLock lock = lockPair.getFirst();
- return new TransactionLock(lock, readonly ? lock.readLock() : lock.writeLock());
- }
-
- public static void removeLock(String project) {
- Pair<ReentrantReadWriteLock, AtomicLong> lockPair = projectLocks.get(project);
- if (lockPair != null) {
- synchronized (UnitOfWork.class) {
- AtomicLong atomicLong = lockPair.getSecond();
- if (atomicLong.decrementAndGet() == 0) {
- projectLocks.remove(project);
- }
-
- }
- }
- if (projectLocks.size() > 5000L) {
- projectLocks.forEach((k, v) -> logger.warn("lock leaks lock: {}，num: {}", k, v.getSecond()));
- }
- }
-
- public static boolean isWriteLocked(String project) {
- ReentrantReadWriteLock lock = projectLocks.get(project).getFirst();
- return lock != null && lock.isWriteLocked();
- }
-
- private ReentrantReadWriteLock rootLock;
-
- @Delegate
- private Lock lock;
-
- public TransactionLock(ReentrantReadWriteLock rootLock, Lock lock) {
- this.rootLock = rootLock;
- this.lock = lock;
- }
-
- boolean isHeldByCurrentThread() {
- if (lock instanceof ReentrantReadWriteLock.ReadLock) {
- return rootLock.getReadHoldCount() > 0;
- } else {
- return rootLock.getWriteHoldCount() > 0;
- }
- }
-
- public static Map<String, ReentrantReadWriteLock> getProjectLocksForRead() {
- Map<String, ReentrantReadWriteLock> map = Maps.newConcurrentMap();
- projectLocks.forEach((k, v) -> map.put(k, v.getFirst()));
- return Collections.unmodifiableMap(map);
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionManagerInstance.java
similarity index 55%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionManagerInstance.java
index 1bbd33831f..267815c501 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionManagerInstance.java
@@ -15,28 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.transaction;
-package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.TransactionLockManager;
+import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionManagerInstance {
+ public static final TransactionLockManager INSTANCE;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+ static {
+ INSTANCE = (TransactionLockManager) ClassUtil
+ .newInstance(KylinConfig.getInstanceFromEnv().getTransactionLockManager());
+ }
- @JsonProperty("task_type")
- private String taskType;
+ private TransactionManagerInstance() {
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public static boolean optimisticEnabled() {
+ return INSTANCE instanceof TransactionOptimisticLockManager;
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java
index 1bbd33831f..9128075b6c 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java
@@ -15,28 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.transaction;
-package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionOptimisticLockManager extends TransactionPessimisticLockManager {
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ @Override
+ public TransactionLock getLock(String project, boolean readonly) {
+ return super.getTempLock(String.valueOf(Thread.currentThread().getId()), readonly);
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java
new file mode 100644
index 0000000000..33ff6fbf3f
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.transaction;
+
+import static org.apache.kylin.common.persistence.lock.LockManagerInstance.INSTANCE;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.persistence.lock.AbstractMemoryLock;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
+import org.apache.kylin.common.persistence.lock.ProjectLock;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.common.persistence.lock.TransactionLockManager;
+import org.apache.kylin.common.persistence.lock.TransactionTempLock;
+
+public class TransactionPessimisticLockManager implements TransactionLockManager {
+
+ @Override
+ public TransactionLock getLock(String project, boolean readonly) {
+ return INSTANCE.getProjectLock(project).getReadOrWriteLock(false);
+ }
+
+ @Override
+ public TransactionLock getTempLock(String project, boolean readonly) {
+ AbstractMemoryLock lock = INSTANCE.getProjectLock(project);
+ return new TransactionTempLock(lock.writeLock());
+ }
+
+ @Override
+ public void removeLock(String project) {
+ AbstractMemoryLock lock = INSTANCE.getProjectLock(project);
+ if (lock != null) {
+ synchronized (UnitOfWork.class) {
+ AtomicLong atomicLong = lock.getTranHoldCount();
+ if (atomicLong.decrementAndGet() == 0) {
+ INSTANCE.removeProjectLock(project);
+ }
+ }
+ }
+ INSTANCE.checkProjectLockSize();
+ }
+
+ public Map<String, ProjectLock> getProjectLocksForRead() {
+ return Collections.unmodifiableMap(INSTANCE.listAllProjectLock());
+ }
+
+ @Override
+ public TransactionLock getProjectLock(String projectName) {
+ return INSTANCE.getProjectLock(projectName).getReadOrWriteLock(false);
+ }
+
+ @Override
+ public TransactionLock getModuleLock(String projectName, ModuleLockEnum moduleName) {
+ return INSTANCE.getModuleLock(projectName, moduleName).getReadOrWriteLock(false);
+ }
+
+ @Override
+ public List<TransactionLock> getPathLock(String pathName, boolean readOnly) {
+ return INSTANCE.getPathLock(pathName).stream().map(pathLock -> pathLock.getReadOrWriteLock(readOnly))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
index f344deb055..f1acfcd5a8 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
@@ -21,6 +21,14 @@ import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
+import static org.apache.kylin.common.persistence.lock.TransactionDeadLockHandler.THREAD_NAME_PREFIX;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.LogConstant;
@@ -39,16 +47,19 @@ import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
import org.apache.kylin.common.persistence.event.ResourceRelatedEvent;
import org.apache.kylin.common.persistence.event.StartUnit;
+import org.apache.kylin.common.persistence.lock.DeadLockException;
+import org.apache.kylin.common.persistence.lock.LockInterruptException;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.TempLock;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.common.persistence.metadata.JdbcMetadataStore;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
@Slf4j
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
@@ -77,33 +88,38 @@ public class UnitOfWork {
}
public static <T> T doInTransactionWithRetry(UnitOfWorkParams<T> params) {
- val maxRetry = params.getMaxRetry();
- val f = params.getProcessor();
- // reused transaction, won't retry
- if (isAlreadyInTransaction()) {
- val unitOfWork = UnitOfWork.get();
- unitOfWork.checkReentrant(params);
- try {
- checkEpoch(params);
- f.preProcess();
- return f.process();
- } catch (Throwable throwable) {
- f.onProcessError(throwable);
- throw new TransactionException("transaction failed due to inconsistent state", throwable);
+ try (SetThreadName ignore = new SetThreadName(THREAD_NAME_PREFIX)) {
+ val f = params.getProcessor();
+ // reused transaction, won't retry
+ if (isAlreadyInTransaction()) {
+ val unitOfWork = UnitOfWork.get();
+ unitOfWork.checkReentrant(params);
+ try {
+ checkEpoch(params);
+ f.preProcess();
+ return f.process();
+ } catch (Throwable throwable) {
+ f.onProcessError(throwable);
+ throw new TransactionException("transaction failed due to inconsistent state", throwable);
+ }
}
- }
- // new independent transaction with retry
- int retry = 0;
- val traceId = RandomUtil.randomUUIDStr();
- while (retry++ < maxRetry) {
+ // new independent transaction with retry
+ int retry = 0;
+ val traceId = RandomUtil.randomUUIDStr();
+ if (params.isRetryMoreTimeForDeadLockException()) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ params.setRetryUntil(System.currentTimeMillis() + config.getMaxSecondsForDeadLockRetry() * 1000);
+ }
+ while (retry++ < params.getMaxRetry()) {
- val ret = doTransaction(params, retry, traceId);
- if (ret.getSecond()) {
- return ret.getFirst();
+ val ret = doTransaction(params, retry, traceId);
+ if (ret.getSecond()) {
+ return ret.getFirst();
+ }
}
+ throw new IllegalStateException("Unexpected doInTransactionWithRetry end");
}
- throw new IllegalStateException("Unexpected doInTransactionWithRetry end");
}
private static <T> Pair<T, Boolean> doTransaction(UnitOfWorkParams<T> params, int retry, String traceId) {
@@ -144,11 +160,15 @@ public class UnitOfWork {
if (isAlreadyInTransaction()) {
try (SetLogCategory ignored = new SetLogCategory(LogConstant.METADATA_CATEGORY)) {
val unitOfWork = UnitOfWork.get();
- unitOfWork.getCurrentLock().unlock();
+ MemoryLockUtils.removeThreadFromGraph();
+ List<TransactionLock> lockInOrder = Arrays
+ .asList(unitOfWork.getCurrentLock().toArray(new TransactionLock[0]));
+ Lists.reverse(lockInOrder).stream().filter(TransactionLock::isHeldByCurrentThread)
+ .forEach(TransactionLock::unlock);
unitOfWork.cleanResource();
- if (params.getTempLockName() != null && !params.getTempLockName().equals(params.getUnitName())) {
- TransactionLock.removeLock(params.getTempLockName());
- }
+ val curLock = UnitOfWork.threadLocals.get().getCurrentLock();
+ curLock.stream().filter(TempLock.class::isInstance)
+ .forEach(l -> TransactionManagerInstance.INSTANCE.removeLock(l.transactionUnit()));
} catch (IllegalStateException e) {
//has not hold the lock yet, it's ok
log.warn(e.getMessage());
@@ -179,23 +199,10 @@ public class UnitOfWork {
}
static <T> UnitOfWorkContext startTransaction(UnitOfWorkParams<T> params) throws Exception {
- val project = params.getUnitName();
+ val unitName = params.getUnitName();
val readonly = params.isReadonly();
checkEpoch(params);
- val lock = params.getTempLockName() == null ? TransactionLock.getLock(project, readonly)
- : TransactionLock.getLock(params.getTempLockName(), readonly);
-
- try (SetLogCategory ignored = new SetLogCategory(LogConstant.METADATA_CATEGORY)) {
- log.trace("get lock for project {}, lock is held by current thread: {}", project, lock.isHeldByCurrentThread());
- }
- //re-entry is not encouraged (because it indicates complex handling logic, bad smell), let's abandon it first
- Preconditions.checkState(!lock.isHeldByCurrentThread());
- lock.lock();
-
- val unitOfWork = new UnitOfWorkContext(project);
- unitOfWork.setCurrentLock(lock);
- unitOfWork.setParams(params);
- threadLocals.set(unitOfWork);
+ final UnitOfWorkContext unitOfWork = initUnitOfContext(params, unitName, readonly);
if (readonly || !params.isUseSandbox()) {
unitOfWork.setLocalConfig(null);
@@ -218,6 +225,34 @@ public class UnitOfWork {
return unitOfWork;
}
+ private static <T> UnitOfWorkContext initUnitOfContext(UnitOfWorkParams<T> params, String unitName,
+ boolean readonly) {
+ val unitOfWork = new UnitOfWorkContext(unitName);
+ TransactionLock lock;
+ if (params.isUseProjectLock()) {
+ lock = TransactionManagerInstance.INSTANCE.getProjectLock(unitName);
+ } else if (params.getTempLockName() != null) {
+ lock = TransactionManagerInstance.INSTANCE.getTempLock(params.getTempLockName(), readonly);
+ } else {
+ lock = TransactionManagerInstance.INSTANCE.getLock(unitName, readonly);
+ if (lock.transactionUnit().equals(unitName)) {
+ params.setUseProjectLock(true);
+ }
+ }
+ try (SetLogCategory ignored = new SetLogCategory(LogConstant.METADATA_CATEGORY)) {
+ log.info("get lock for project {}, lock is held by current thread: {}", unitName,
+ lock.isHeldByCurrentThread());
+ }
+ // re-entry is not encouraged (because it indicates complex handling logic, bad smell), let's abandon it first
+ Preconditions.checkState(!lock.isHeldByCurrentThread());
+ lock.lock();
+ unitOfWork.getCurrentLock().add(lock);
+
+ unitOfWork.setParams(params);
+ threadLocals.set(unitOfWork);
+ return unitOfWork;
+ }
+
private static <T> void checkEpoch(UnitOfWorkParams<T> params) throws Exception {
val checker = params.getEpochChecker();
if (checker != null && !params.isReadonly()) {
@@ -241,6 +276,16 @@ public class UnitOfWork {
}
val threadViewRS = (ThreadViewResourceStore) ResourceStore.getKylinMetaStore(config);
List<RawResource> data = threadViewRS.getResources();
+ if (!params.isUseProjectLock()
+ && (threadViewRS.getMetadataStore() instanceof JdbcMetadataStore || config.isUTEnv())) {
+ data.forEach(rawResource -> {
+ if (!UnitOfWork.checkWriteLock(rawResource.getResPath())) {
+ throw new IllegalStateException(
+ "Transaction does not hold a write lock for resPath: " + rawResource.getResPath());
+ }
+ });
+ }
+
val eventList = data.stream().map(x -> {
if (x instanceof TombRawResource) {
return new ResourceDeleteEvent(x.getResPath());
@@ -280,12 +325,37 @@ public class UnitOfWork {
}
}
+ public static void recordLocks(String resPath, TransactionLock lock, boolean readOnly) {
+ Preconditions.checkState(lock.isHeldByCurrentThread());
+ UnitOfWork.get().getCurrentLock().add(lock);
+ if (readOnly) {
+ UnitOfWork.get().getReadLockPath().add(resPath);
+ } else {
+ UnitOfWork.get().getWriteLockPath().add(resPath);
+ }
+ }
+
+ public static boolean checkWriteLock(String resPath) {
+ return UnitOfWork.get().getWriteLockPath().contains(resPath);
+ }
+
private static void handleError(Throwable throwable, UnitOfWorkParams<?> params, int retry, String traceId) {
if (throwable instanceof KylinException && Objects.nonNull(((KylinException) throwable).getErrorCodeProducer())
&& ((KylinException) throwable).getErrorCodeProducer().getErrorCode()
.equals(ErrorCodeSystem.MAINTENANCE_MODE_WRITE_FAILED.getErrorCode())) {
retry = params.getMaxRetry();
}
+ if (throwable instanceof DeadLockException) {
+ log.debug("DeadLock found in this transaction, will retry");
+ if (params.isRetryMoreTimeForDeadLockException() && System.currentTimeMillis() < params.getRetryUntil()) {
+ params.setMaxRetry(params.getMaxRetry() + 1);
+ }
+ }
+ if (throwable instanceof LockInterruptException) {
+ // Just remove the interrupted flag.
+ Thread.interrupted();
+ log.debug("DeadLock is found by TransactionDeadLockHandler, will retry");
+ }
if (throwable instanceof QuitTxnRightNow) {
retry = params.getMaxRetry();
}
@@ -333,7 +403,6 @@ public class UnitOfWork {
public static boolean isReplaying() {
return Objects.equals(true, replaying.get());
-
}
public static boolean isReadonly() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java
index e546c62127..08083664d6 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java
@@ -17,13 +17,17 @@
*/
package org.apache.kylin.common.persistence.transaction;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.CommonErrorCode;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
-
+import org.apache.kylin.common.persistence.lock.TransactionLock;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
@@ -42,7 +46,9 @@ public class UnitOfWorkContext {
private final String project;
private KylinConfig.SetAndUnsetThreadLocalConfig localConfig;
- private TransactionLock currentLock = null;
+ private Set<TransactionLock> currentLock = new LinkedHashSet<>();
+ private Set<String> writeLockPath = new HashSet<>();
+ private Set<String> readLockPath = new HashSet<>();
@Delegate
private UnitOfWorkParams params;
@@ -51,6 +57,8 @@ public class UnitOfWorkContext {
List<UnitTask> onUpdatedTasks = Lists.newArrayList();
+ List<UnitTask> onStartUpdateTasks = Lists.newArrayList();
+
public void doAfterUnit(UnitTask task) {
onFinishedTasks.add(task);
}
@@ -59,6 +67,10 @@ public class UnitOfWorkContext {
onUpdatedTasks.add(task);
}
+ public void doBeforeUpdate(UnitTask task) {
+ onStartUpdateTasks.add(task);
+ }
+
void cleanResource() {
if (localConfig == null) {
return;
@@ -71,9 +83,9 @@ public class UnitOfWorkContext {
}
void checkLockStatus() {
- Preconditions.checkNotNull(currentLock);
- Preconditions.checkState(currentLock.isHeldByCurrentThread());
-
+ Preconditions.checkState(CollectionUtils.isNotEmpty(currentLock));
+ // Some readLocks have been released when upgrade to writeLock.
+ Preconditions.checkState(currentLock.stream().anyMatch(TransactionLock::isHeldByCurrentThread));
}
void checkReentrant(UnitOfWorkParams params) {
@@ -109,6 +121,12 @@ public class UnitOfWorkContext {
});
}
+ public void onStartUnitUpdate() throws Exception{
+ for (UnitTask task : onStartUpdateTasks) {
+ task.run();
+ }
+ }
+
public interface UnitTask {
void run() throws Exception;
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
index 57e339fe37..bb7602847d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
@@ -17,12 +17,17 @@
*/
package org.apache.kylin.common.persistence.transaction;
+import java.util.List;
import java.util.function.Consumer;
import org.apache.kylin.common.persistence.event.ResourceRelatedEvent;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
@Data
@Builder
@@ -34,6 +39,10 @@ public class UnitOfWorkParams<T> {
private Consumer<ResourceRelatedEvent> writeInterceptor;
+ private UnitRetryContext retryContext;
+
+ private long retryUntil;
+
@Builder.Default
private boolean all = false;
@@ -58,6 +67,12 @@ public class UnitOfWorkParams<T> {
private boolean skipAuditLog = false;
private String tempLockName;
+
+ @Builder.Default
+ private boolean useProjectLock = false;
+
+ @Builder.Default
+ private boolean retryMoreTimeForDeadLockException = false;
/**
* only for debug or test
@@ -65,4 +80,13 @@ public class UnitOfWorkParams<T> {
@Builder.Default
private long sleepMills = -1;
+ @Getter
+ @Setter
+ @AllArgsConstructor
+ public static class UnitRetryContext {
+ private List<TransactionLock> retryLock;
+ private boolean allowRetryNext;
+ private boolean optimisticLockEnabled;
+ }
+
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index 78513f1d74..7211d629cb 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -38,6 +38,10 @@ import java.util.function.BiConsumer;
import javax.annotation.Nullable;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.type.TypeFactory;
+import com.fasterxml.jackson.databind.util.LRUMap;
+import com.fasterxml.jackson.databind.util.LookupCache;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
@@ -59,13 +63,17 @@ public class JsonUtil {
.setFailOnUnknownId(false);
static {
+ LookupCache<Object, JavaType> cache = new LRUMap<>(16, 2000);
+ TypeFactory customTypeFactory = TypeFactory.defaultInstance().withCache(cache);
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.setConfig(mapper.getSerializationConfig().withView(PersistenceView.class));
mapper.setFilterProvider(simpleFilterProvider);
+ mapper.setTypeFactory(customTypeFactory);
indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true)
.setConfig(indentMapper.getSerializationConfig().withView(PersistenceView.class));
indentMapper.setFilterProvider(simpleFilterProvider);
+ indentMapper.setTypeFactory(customTypeFactory);
}
public static <T> T readValue(File src, Class<T> valueType) throws IOException {
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 2e19879659..408746be99 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -216,6 +216,9 @@ KE-040027208=参数 “%s” 不可为空, 在 “%s” 中。请检查后重试
KE-040027209=无法绑定项目 “%s” 的资源请求。请确保一个项目最多绑定两个资源组，且构建和查询请求各绑定一个资源组。
KE-040027210=在 “mapping_info” 中参数 “resource_group_id” 的值 “%s”，与 “resource_groups” 中的值不匹配。
+## 400282XX deadlock
+KE-040028201=“%s” 不是死锁线程 ID。
+
# Tool
## 500402XX
KE-050040201=参数 “%s” 为空。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index f9d66504ca..4bdfe174ba 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -216,6 +216,9 @@ KE-040027208=The parameter "%s" cannot be empty in "%s". Please check and try ag
KE-040027209=Can't allocate resource group for project "%s". Please ensure that the project is allocated to two resource groups at most. Meanwhile, each request (query or build) is allocated to one resource group.
KE-040027210=The value "%s" of "resource_group_id" in "mapping_info" does not match the value of "resource_groups".
+## 400282XX deadlock
+KE-040028201="%s" are not deadlock thread IDs.
+
# Tool
## 500402XX
KE-050040201="%s" is empty.
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index f8dd010445..f0162f4e95 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -222,6 +222,9 @@ KE-040027208
KE-040027209
KE-040027210
+## 400282XX deadlock
+KE-040028201
+
# Segment
## 100222XX Segment
KE-010022218
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java
index b667b2643b..33749e8212 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java
@@ -27,11 +27,16 @@ import java.util.stream.IntStream;
import org.apache.commons.dbcp2.BasicDataSourceFactory;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.JdbcAuditLogStore;
import org.apache.kylin.common.persistence.transaction.AuditLogReplayWorker;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.scheduler.EventBusFactory;
+import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.base.Joiner;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.awaitility.Awaitility;
@@ -42,11 +47,6 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.DefaultTransactionDefinition;
-import org.apache.kylin.guava30.shaded.common.base.Joiner;
-
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -97,12 +97,14 @@ public class JdbcAuditLogRecoveryTest {
val txManager = auditLogStore.getTransactionManager();
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/_global/project/p1.json", null, false);
store.checkAndPutResource("/_global/project/p1.json", new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
return null;
}, "p1");
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/_global/project/p2.json", null, false);
store.checkAndPutResource("/_global/project/p2.json", new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
return null;
@@ -119,6 +121,7 @@ public class JdbcAuditLogRecoveryTest {
Thread.sleep(500);
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
val path = "/p1/abc-" + System.currentTimeMillis();
+ MemoryLockUtils.lockAndRecord(path, null, false);
val originAbc = store.getResource(path);
store.checkAndPutResource(path, ByteSource.wrap("abc".getBytes(charset)),
System.currentTimeMillis(), originAbc == null ? -1 : originAbc.getMvcc());
@@ -127,7 +130,7 @@ public class JdbcAuditLogRecoveryTest {
try {
auditLogStore.catchupWithTimeout();
} catch (Exception e) {
- log.debug("catchup 1st phase failed", e);
+ log.error("catchup 1st phase failed", e);
}
});
t1.start();
@@ -178,13 +181,14 @@ public class JdbcAuditLogRecoveryTest {
try {
auditLogStore.catchupWithTimeout();
} catch (Exception e) {
- log.debug("catchup 2nd phase failed", e);
+ log.error("catchup 2nd phase failed", e);
}
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
IntStream.range(1000, 1000 + size).forEach(id -> {
String path = "/p2/abc" + id;
+ MemoryLockUtils.lockAndRecord(path, null, false);
val originAbc = store.getResource(path);
store.checkAndPutResource(path, ByteSource.wrap((path + "-version2").getBytes(charset)),
System.currentTimeMillis(), originAbc == null ? -1 : originAbc.getMvcc());
@@ -194,10 +198,10 @@ public class JdbcAuditLogRecoveryTest {
try {
auditLogStore.catchupWithTimeout();
} catch (Exception e) {
- log.debug("catchup 3rd phase failed", e);
+ log.error("catchup 3rd phase failed", e);
}
}).start();
- Awaitility.await().atMost(20, TimeUnit.SECONDS).until(() -> listener.status == -1);
+ Awaitility.await().atMost(200, TimeUnit.SECONDS).until(() -> listener.status == -1);
Assert.assertEquals(203, systemStore.listResourcesRecursively("/").size());
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java
index 696791f601..ee48dc8b4e 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java
@@ -31,28 +31,32 @@ import java.util.stream.IntStream;
import org.apache.commons.dbcp2.BasicDataSourceFactory;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.persistence.event.Event;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.MetadataStore;
import org.apache.kylin.common.persistence.metadata.jdbc.RawResourceRowMapper;
+import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.junit.Assert;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
+import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
@MetadataInfo(onlyProps = true)
@OverwriteProp(key = "kylin.metadata.url", value = "test@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1,username=sa,password=")
-public class JdbcMetadataStoreTest {
+class JdbcMetadataStoreTest {
public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
@@ -63,9 +67,13 @@ public class JdbcMetadataStoreTest {
}
@Test
- public void testBasic() throws IOException {
+ void testBasic() throws IOException {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc2", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc3", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc4", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
store.checkAndPutResource("/p1/abc2", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
store.checkAndPutResource("/p1/abc3", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
@@ -97,9 +105,10 @@ public class JdbcMetadataStoreTest {
}
@Test
- public void testReload() throws Exception {
+ void testReload() throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/_global/abc", null, false);
store.checkAndPutResource("/_global/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
return 0;
}, "_global");
@@ -119,7 +128,7 @@ public class JdbcMetadataStoreTest {
}
@Test
- public void testPage() throws Exception {
+ void testPage() throws Exception {
val dataSource = new DriverManagerDataSource();
val url = getTestConfig().getMetadataUrl();
dataSource.setUrl(url.getParameter("url"));
@@ -156,9 +165,10 @@ public class JdbcMetadataStoreTest {
@Test
@Disabled("for develop")
- public void testDuplicate() {
+ void testDuplicate() {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), 0);
return 0;
@@ -175,13 +185,14 @@ public class JdbcMetadataStoreTest {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), 1);
return 0;
}, "p1");
}
@Test
- public void testBatchUpdate() throws Exception {
+ void testBatchUpdate() throws Exception {
val metadataStore = MetadataStore.createMetadataStore(getTestConfig());
List<Event> events = Collections.singletonList(new ResourceCreateOrUpdateEvent(new RawResource("/p1/test",
ByteSource.wrap("test content".getBytes(StandardCharsets.UTF_8)), System.currentTimeMillis(), 0)));
@@ -223,12 +234,26 @@ public class JdbcMetadataStoreTest {
CompressionUtils.decompress(auditLogContents));
}
+ @Test
+ void testUpdateTheDeletedResource() {
+ val metadataStore = MetadataStore.createMetadataStore(getTestConfig());
+ ReflectionTestUtils.setField(metadataStore, "isUT", false);
+ List<Event> events = Collections.singletonList(new ResourceCreateOrUpdateEvent(new RawResource("/p1/test",
+ ByteSource.wrap("test content".getBytes(StandardCharsets.UTF_8)), System.currentTimeMillis(), 4)));
+ val unitMessages = new UnitMessages(events);
+
+ Assertions.assertThrows(TransactionException.class, () -> UnitOfWork.doInTransactionWithRetry(() -> {
+ metadataStore.batchUpdate(unitMessages, false, "/p1/test", -1);
+ return null;
+ }, "p1", 1));
+ }
+
@OverwriteProp.OverwriteProps({ //
@OverwriteProp(key = "kylin.metadata.compress.enabled", value = "false"), //
@OverwriteProp(key = "kylin.server.port", value = "8081")//
})
@Test
- public void testBatchUpdateWithMetadataCompressDisable() throws Exception {
+ void testBatchUpdateWithMetadataCompressDisable() throws Exception {
val metadataStore = MetadataStore.createMetadataStore(getTestConfig());
List<Event> events = Collections.singletonList(new ResourceCreateOrUpdateEvent(new RawResource("/p1/test",
ByteSource.wrap("test content".getBytes(StandardCharsets.UTF_8)), System.currentTimeMillis(), 0)));
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java
index b984d67d38..44303f4781 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java
@@ -22,17 +22,19 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.DeadLockException;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
+import org.apache.kylin.guava30.shaded.common.base.Throwables;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.base.Throwables;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
@MetadataInfo(onlyProps = true)
@@ -42,19 +44,22 @@ public class UnitOfWorkTest {
public void testTransaction() {
val ret = UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res",
+ MemoryLockUtils.lockAndRecord("/_global/path/res", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res2", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res3", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res2",
+ resourceStore.checkAndPutResource("/_global/path/res2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res3",
+ resourceStore.checkAndPutResource("/_global/path/res3",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}, UnitOfWork.GLOBAL_UNIT);
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res2").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res3").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res2").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res3").getMvcc());
}
@Test
@@ -62,9 +67,11 @@ public class UnitOfWorkTest {
try {
val ret = UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res",
+ MemoryLockUtils.lockAndRecord("/_global/path/res", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res2", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res2",
+ resourceStore.checkAndPutResource("/_global/path/res2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
throw new IllegalArgumentException("surprise");
}, UnitOfWork.GLOBAL_UNIT);
@@ -72,8 +79,8 @@ public class UnitOfWorkTest {
}
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertNull(resourceStore.getResource("/_global/path/to/res"));
- Assert.assertNull(resourceStore.getResource("/_global/path/to/res2"));
+ Assert.assertNull(resourceStore.getResource("/_global/path/res"));
+ Assert.assertNull(resourceStore.getResource("/_global/path/res2"));
// test can be used again after exception
testTransaction();
@@ -132,45 +139,53 @@ public class UnitOfWorkTest {
public void testReentrant() {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res",
+ MemoryLockUtils.lockAndRecord("/_global/path/res", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res2", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res2",
+ resourceStore.checkAndPutResource("/_global/path/res2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore2 = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore2.checkAndPutResource("/_global/path2/to/res2/1",
+ MemoryLockUtils.lockAndRecord("/_global/path2/res2/1", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path2/res2/2", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path2/res2/3", null, false);
+ resourceStore2.checkAndPutResource("/_global/path2/res2/1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore2.checkAndPutResource("/_global/path2/to/res2/2",
+ resourceStore2.checkAndPutResource("/_global/path2/res2/2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore2.checkAndPutResource("/_global/path2/to/res2/3",
+ resourceStore2.checkAndPutResource("/_global/path2/res2/3",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
Assert.assertEquals(resourceStore, resourceStore2);
return 0;
}, UnitOfWork.GLOBAL_UNIT);
- resourceStore.checkAndPutResource("/_global/path/to/res3",
+ MemoryLockUtils.lockAndRecord("/_global/path/res3", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res3",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}, UnitOfWork.GLOBAL_UNIT);
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res2").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path2/to/res2/1").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path2/to/res2/2").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path2/to/res2/3").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res3").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res2").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path2/res2/1").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path2/res2/2").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path2/res2/3").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res3").getMvcc());
}
@Test
public void testReadLockExclusive() {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res1",
+ resourceStore.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
Object condition = new Object();
AtomicBoolean stop = new AtomicBoolean();
Thread readLockHelder = new Thread(() -> {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(true).maxRetry(1).processor(() -> {
+ ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
+ .getResource("/_global/path/res1");
synchronized (condition) {
condition.notify();
}
@@ -203,7 +218,7 @@ public class UnitOfWorkTest {
long cost = System.currentTimeMillis() - readStart;
Assert.assertTrue(cost < 500);
Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ .getResource("/_global/path/res1").getMvcc());
return 0;
}).build());
} catch (Exception e) {
@@ -221,10 +236,11 @@ public class UnitOfWorkTest {
try {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(false).maxRetry(1).processor(() -> {
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
long cost = System.currentTimeMillis() - writeStart;
Assert.assertTrue(cost > 1500);
Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ .getResource("/_global/path/res1").getMvcc());
return 0;
}).build());
} catch (Exception e) {
@@ -242,7 +258,9 @@ public class UnitOfWorkTest {
.readonly(false).maxRetry(1).processor(() -> {
val resourceStoreInTransaction = ResourceStore
.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStoreInTransaction.checkAndPutResource("/_global/path/to/res1",
+ System.out.println("Write thread start to lock");
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
+ resourceStoreInTransaction.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
synchronized (condition) {
condition.notify();
@@ -261,6 +279,7 @@ public class UnitOfWorkTest {
synchronized (condition) {
condition.notify();
}
+ System.out.println("Write thread finished.");
return 0;
}).build());
});
@@ -284,14 +303,18 @@ public class UnitOfWorkTest {
try {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(true).maxRetry(1).processor(() -> {
+ System.out.println("Read thread start to lock.");
+ RawResource raw = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
+ .getResource("/_global/path/res1");
+ System.out.println("Read thread lock succeed.");
long cost = System.currentTimeMillis() - start;
+ System.out.println(cost + " " + (raw == null ? -2 : raw.getMvcc()));
Assert.assertTrue(cost > 1500);
- Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ Assert.assertEquals(0, raw.getMvcc());
return 0;
}).build());
} catch (Exception e) {
- Assert.fail();
+ throw e;
}
stop.set(true);
}
@@ -303,7 +326,8 @@ public class UnitOfWorkTest {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(true).maxRetry(1).processor(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res1",
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}).build());
@@ -315,12 +339,12 @@ public class UnitOfWorkTest {
@Test
public void testReadTransaction() {
- ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv()).checkAndPutResource("/_global/path/to/res1",
+ ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv()).checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
UnitOfWork.doInTransactionWithRetry(
UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT).readonly(true).maxRetry(1).processor(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res1").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res1").getMvcc());
return 0;
}).build());
}
@@ -331,12 +355,27 @@ public class UnitOfWorkTest {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT).readonly(false)
.maxRetry(1).processor(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res1",
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}).build());
Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ .getResource("/_global/path/res1").getMvcc());
+
+ }
+ @Test
+ void testRetryMoreTimeForDeadLockException() {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.env.max-seconds-for-dead-lock-retry", "2");
+ long startTime = System.currentTimeMillis();
+ try {
+ UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().retryMoreTimeForDeadLockException(true).processor(() -> {
+ throw new DeadLockException("test");
+ }).build());
+ } catch (Exception e) {
+ Assert.assertEquals(DeadLockException.class, e.getCause().getClass());
+ Assert.assertTrue(System.currentTimeMillis() - startTime > 2 * 1000);
+ }
}
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java
new file mode 100644
index 0000000000..39484eba23
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import static org.awaitility.Awaitility.await;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.InMemResourceStore;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ThreadViewResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@MetadataInfo(onlyProps = true)
+@Disabled("Only run this manually")
+class CloseWaitTest {
+
+ private Server server = null;
+ private static ResourceStore store = null;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.env.lock-timeout", "30");
+ config.setProperty("kylin.env.dead-lock-check-interval", "10");
+
+ if (server == null) {
+ server = new Server(7077);
+ server.setHandler(new TransactionHandler());
+ server.start();
+ InMemResourceStore inMemResourceStore = (InMemResourceStore) ResourceStore.getKylinMetaStore(config);
+ store = new ThreadViewResourceStore(inMemResourceStore, config);
+ TransactionDeadLockHandler.getInstance().start();
+ }
+ }
+
+ @AfterEach
+ public void cleanup() throws Exception {
+ server.stop();
+ }
+
+ private static class TransactionHandler extends AbstractHandler {
+
+ @Override
+ public void handle(String s, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
+ throws IOException, ServletException {
+ response.setStatus(HttpServletResponse.SC_OK);
+ response.getWriter().println("Mocked response");
+ baseRequest.setHandled(true);
+ if (s.equals("/api/suggest")) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.manuallyLockModule("default", ModuleLockEnum.MODEL, store);
+ System.out.println("Suggest transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Suggest transaction failed");
+ }
+ } else if (s.equals("/api/drop")) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", false, store, () -> null);
+ System.out.println("Drop transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Drop transaction failed");
+ }
+ }
+ }
+ }
+
+ @Disabled("Only run this manually")
+ @Test
+ void closeWaitTest() {
+ AtomicBoolean shouldContinue = new AtomicBoolean(true);
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(5* 1000).setConnectionRequestTimeout(5 *1000)
+ .setSocketTimeout(5*1000).build();
+ Runnable suggest = () -> {
+ while (shouldContinue.get()) {
+ long start = System.currentTimeMillis();
+ try (CloseableHttpClient client = HttpClientBuilder.create()
+ .setDefaultRequestConfig(requestConfig).build()) {
+ HttpPost suggestRequest = new HttpPost("http://127.0.0.1:7077/api/suggest");
+
+ client.execute(suggestRequest);
+ } catch (IOException e) {
+ System.out.println("suggestRequest failed.");
+ } finally {
+ System.out.println("suggestRequest cost: " + (System.currentTimeMillis() - start) + "ms");
+ }
+ }
+ };
+ Runnable drop = () -> {
+ while (shouldContinue.get()) {
+ long start = System.currentTimeMillis();
+ try (CloseableHttpClient client = HttpClientBuilder.create()
+ .setDefaultRequestConfig(requestConfig).build()) {
+ HttpPost suggestRequest = new HttpPost("http://127.0.0.1:7077/api/drop");
+
+ client.execute(suggestRequest);
+ } catch (IOException e) {
+ System.out.println("dropRequest failed.");
+ } finally {
+ System.out.println("suggestRequest cost: " + (System.currentTimeMillis() - start) + "ms");
+ }
+ }
+ };
+ List<Thread> threads = new ArrayList<>();
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(drop));
+
+ threads.forEach(Thread::start);
+ await().atMost(10000, TimeUnit.SECONDS).until(() -> false);
+ }
+
+}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java
new file mode 100644
index 0000000000..c25ab05236
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import static org.awaitility.Awaitility.await;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.InMemResourceStore;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ThreadViewResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@MetadataInfo(onlyProps = true)
+class MemoryLockGraphTest {
+
+ @Test
+ @Disabled("A simulation of long running. When the number of CPU cores is small, it mat fail.")
+ void deadLockWithParallelThreadTest() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ InMemResourceStore inMemResourceStore = (InMemResourceStore) ResourceStore.getKylinMetaStore(config);
+ ThreadViewResourceStore store = new ThreadViewResourceStore(inMemResourceStore, config);
+ AtomicBoolean shouldContinue = new AtomicBoolean(true);
+ Runnable suggest = () -> {
+ while(shouldContinue.get()) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.manuallyLockModule("default", ModuleLockEnum.MODEL, store);
+ System.out.println("Suggest transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Suggest transaction failed");
+ }
+ }
+ };
+ Runnable drop = () -> {
+ while(shouldContinue.get()) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", false, store, () -> null);
+ System.out.println("Drop transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Drop transaction failed");
+ }
+ }
+ };
+
+ List<Thread> threads = new ArrayList<>();
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(drop));
+
+
+ config.setProperty("kylin.env.dead-lock-check-interval", "3");
+ TransactionDeadLockHandler.getInstance().start();
+
+ threads.forEach(Thread::start);
+
+ await().atMost(60, TimeUnit.SECONDS).until(() -> threads.stream().allMatch(t -> t.getState() == Thread.State.TIMED_WAITING));
+ await().atMost(4, TimeUnit.SECONDS).until(() -> threads.stream().anyMatch(t -> t.getState() != Thread.State.TIMED_WAITING));
+
+ shouldContinue.set(false);
+ }
+
+ @Test
+ void deadLockCheckTest() throws InterruptedException {
+ MemoryLockGraph graph = new MemoryLockGraph();
+ MockedLockManager lockManager = new MockedLockManager(5, graph);
+ lockManager.addLock(graph, "R_r5", "R_r1", "R_r2", "R_r3", "R_r3,R_r4");
+ lockManager.addLock(graph, "W_r1", "W_r2", "W_r3", "W_r4", "W_r5");
+ List<List<Long>> cycles = graph.checkForDeadLock();
+ cycles.sort((l1, l2) -> l2.size() - l1.size());
+ Assertions.assertEquals(2, cycles.size());
+ List<Long> ids = lockManager.getThreadIds();
+ Assertions.assertEquals(ids.size(), cycles.get(0).size());
+ Assertions.assertTrue(cycles.get(0).containsAll(ids));
+ Assertions.assertEquals(4, cycles.get(1).size());
+ Assertions.assertTrue(cycles.get(1).containsAll(Arrays.asList(ids.get(0), ids.get(1), ids.get(2), ids.get(4))));
+ lockManager.interruptAllThread();
+ }
+
+ @Test
+ void getKeyNodeForDeadLockTest() throws InterruptedException {
+ MemoryLockGraph graph = new MemoryLockGraph();
+ MockedLockManager lockManager = new MockedLockManager(5, graph);
+ lockManager.addLock(graph, "", "R_r4,R_r1", "R_r4,R_r2", "", "");
+ lockManager.addLock(graph, "", "", "", "R_r3,R_r5", "R_r3");
+ lockManager.addLock(graph, "W_r1", "W_r2", "W_r3", "W_r4", "W_r5");
+ List<List<Long>> cycles = graph.checkForDeadLock();
+ cycles.sort((l1, l2) -> l2.size() - l1.size());
+ Assertions.assertEquals(4, cycles.size());
+ Set<Long> nodesInCycle = graph.getKeyNodes(cycles);
+ List<Long> ids = lockManager.getThreadIds();
+ Assertions.assertArrayEquals(new Long[] { ids.get(3) }, nodesInCycle.toArray());
+ lockManager.interruptAllThread();
+ }
+
+ @Test
+ void preCheckTest() throws InterruptedException {
+ MemoryLockGraph graph = new MemoryLockGraph();
+ MockedLockManager lockManager = new MockedLockManager(5, graph);
+ lockManager.addLock(graph, "", "R_r1", "R_r2", "R_r3", "R_r3");
+ lockManager.addLock(graph, "W_r1", "W_r2", "W_r3", "", "");
+ List<Long> allThreads = lockManager.getThreadIds();
+
+ List<Long> cycle;
+ cycle = graph.preCheck(allThreads.get(4), new HashSet<>(Collections.singletonList(allThreads.get(1))));
+ Assertions.assertFalse(cycle.isEmpty());
+
+ cycle = graph.preCheck(allThreads.get(0), new HashSet<>(Collections.singletonList(allThreads.get(3))));
+ Assertions.assertTrue(cycle.isEmpty());
+
+ lockManager.interruptAllThread();
+ }
+
+ static class MockedLockManager {
+ List<Thread> threads = new ArrayList<>();
+ List<ArrayBlockingQueue<String>> lockQueues = new ArrayList<>();
+
+ public MockedLockManager(int cnt, MemoryLockGraph graph) {
+ for (int i = 0; i < cnt; i++) {
+ lockQueues.add(new ArrayBlockingQueue<>(10));
+ int finalI = i;
+ Thread t = new Thread(() -> {
+ long threadId = Thread.currentThread().getId();
+ String res;
+ List<TransactionLock> holdLocks = new ArrayList<>();
+ while (true) {
+ try {
+ res = lockQueues.get(finalI).take();
+ if (res.isEmpty()) {
+ continue;
+ }
+ String path = "/" + res.substring(2);
+ List<TransactionLock> locks = MemoryLockUtils.getPathLocks(path, res.startsWith("R"));
+ locks.forEach(lock -> {
+ graph.setThread(threadId, lock);
+ lock.lock();
+ graph.resetThread(threadId);
+ holdLocks.add(lock);
+ });
+ } catch (Exception ignored) {
+ holdLocks.forEach(TransactionLock::unlock);
+ break;
+ }
+ }
+ });
+ t.start();
+ this.threads.add(t);
+ }
+ }
+
+ public void addLock(MemoryLockGraph graph, String... resourceToLock) throws InterruptedException {
+ for (int i = 0; i < resourceToLock.length; i++) {
+ for (String res : resourceToLock[i].split(",")) {
+ this.lockQueues.get(i).add(res);
+ }
+ }
+ // wait thread to take
+ for (int i = 0; i < threads.size(); i++) {
+ int finalI = i;
+ await().atMost(1, TimeUnit.SECONDS).until(() -> lockQueues.get(finalI).isEmpty()
+ || graph.isThreadWaitForLock(threads.get(finalI).getId()));
+ }
+ }
+
+ public void interruptAllThread() {
+ this.threads.forEach(Thread::interrupt);
+ }
+
+ public List<Long> getThreadIds() {
+ return this.threads.stream().map(Thread::getId).collect(Collectors.toList());
+ }
+ }
+}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java
new file mode 100644
index 0000000000..84f54c33bd
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.MODEL;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.TABLE;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class LockEliminationRuleExecTest extends NLocalFileMetadataTestCase {
+
+ @BeforeEach
+ public void setup() throws Exception {
+ createTestMetadata();
+ }
+
+ @AfterEach
+ public void teardown() {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ void testProjectRule() {
+ // ProjectLock ModuleLock PathLock
+ // Source: default TABLE table1,table2
+ // Expected: default NULL NULL
+
+ String project = "default";
+ {
+ List<TransactionLock> projectLock = MemoryLockUtils.getProjectLock(project);
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertFalse(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(project, lockInfo.getProjectLocks().get(0).transactionUnit());
+ }
+ {
+ List<TransactionLock> projectLock = MemoryLockUtils.getProjectLock(project);
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, TABLE);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertFalse(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(project, lockInfo.getProjectLocks().get(0).transactionUnit());
+ }
+ {
+ List<TransactionLock> projectLock = MemoryLockUtils.getProjectLock(project);
+ List<TransactionLock> moduleLocks = Collections.emptyList();
+ List<TransactionLock> pathLocks = Collections.emptyList();
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertFalse(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(project, lockInfo.getProjectLocks().get(0).transactionUnit());
+ }
+ }
+
+ @Test
+ void testModuleRule() {
+ // ProjectLock ModuleLock PathLock
+ // Source: NULL TABLE table1,table2
+ // Expected: NULL TABLE NULL
+ String project = "default";
+
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, TABLE);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(TABLE, lockInfo.getModuleLocks().get(0).moduleEnum());
+ }
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(MODEL, lockInfo.getModuleLocks().get(0).moduleEnum());
+ Assertions.assertEquals(2, lockInfo.getPathLocks().size());
+ }
+ }
+
+ @Test
+ void testPathRule() {
+ // ProjectLock ModuleLock PathLock
+ // Source: NULL MODEL table1,table2
+ // Expected: NULL MODEL,TABLE NULL
+ String project = "default";
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.env.lock-remove-max-size", "1");
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+
+ Assertions.assertEquals(2, lockInfo.getModuleLocks().size());
+ }
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils
+ .getPathLocks(Lists.newArrayList("/default/table/db.table1.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getPathLocks().isEmpty());
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java
index deec6c65a1..9626449610 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java
@@ -43,11 +43,13 @@ import org.apache.kylin.common.persistence.StringEntity;
import org.apache.kylin.common.persistence.UnitMessages;
import org.apache.kylin.common.persistence.event.Event;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.jdbc.AuditLogRowMapper;
import org.apache.kylin.common.persistence.transaction.AuditLogReplayWorker;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.JdbcInfo;
import org.apache.kylin.junit.annotation.JdbcMetadataInfo;
import org.apache.kylin.junit.annotation.MetadataInfo;
@@ -62,7 +64,6 @@ import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -78,6 +79,10 @@ class JdbcAuditLogStoreTest {
void testUpdateResourceWithLog(JdbcInfo info) throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc2", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc3", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc4", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(charset)), -1);
store.checkAndPutResource("/p1/abc2", ByteSource.wrap("abc".getBytes(charset)), -1);
store.checkAndPutResource("/p1/abc3", ByteSource.wrap("abc".getBytes(charset)), -1);
@@ -476,6 +481,7 @@ class JdbcAuditLogStoreTest {
public void testGet() throws IOException {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/123", null, false);
store.checkAndPutResource("/p1/123", ByteSource.wrap("123".getBytes(charset)), -1);
return 0;
}, "p1");
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionLockTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionProjectLockTest.java
similarity index 89%
rename from src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionLockTest.java
rename to src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionProjectLockTest.java
index 06c35d8113..ce86a92486 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionLockTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionProjectLockTest.java
@@ -19,23 +19,23 @@ package org.apache.kylin.common.persistence.transaction;
import java.util.concurrent.TimeUnit;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.assertj.core.util.Lists;
import org.awaitility.Awaitility;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class TransactionLockTest {
+@MetadataInfo(onlyProps = true)
+class TransactionProjectLockTest {
@Test
- public void testStravition() {
+ void testStravition() {
val threads = Lists.<LogicThread> newArrayList();
for (int i = 0; i < 10; i++) {
@@ -51,7 +51,6 @@ public class TransactionLockTest {
long minCount = threads.stream().mapToLong(LogicThread::getCount).min().getAsLong();
Assert.assertTrue(maxCount - minCount <= 1);
- TransactionLock.projectLocks = Maps.newConcurrentMap();
for (LogicThread thread : threads) {
thread.interrupt();
}
@@ -69,12 +68,12 @@ public class TransactionLockTest {
@Override
public void run() {
super.run();
- val tLock = TransactionLock.getLock("p0", isReader);
+ val tLock = TransactionManagerInstance.INSTANCE.getLock("p0", isReader);
while (count < 20) {
tLock.lock();
log.debug("lock by {} {}, {}", isReader ? "reader" : "writer", Thread.currentThread().getName(), count);
try {
- Thread.sleep(100);
+ TimeUnit.MILLISECONDS.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
index 0f58537aa7..5fc9a48e00 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
@@ -27,6 +27,7 @@ import java.util.Objects;
import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -104,8 +105,8 @@ public class SegmentUtil {
protected static <T extends ISegment> boolean anyIndexJobRunning(T segment) {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
NExecutableManager execManager = NExecutableManager.getInstance(kylinConfig, segment.getModel().getProject());
- val executables = execManager.listExecByJobTypeAndStatus(ExecutableState::isRunning, INDEX_BUILD,
- SUB_PARTITION_BUILD);
+ val executables = execManager.listPartialExec(path -> StringUtils.endsWith(path, segment.getModel().getId()),
+ ExecutableState::isRunning, INDEX_BUILD, SUB_PARTITION_BUILD);
return executables.stream().anyMatch(task -> task.getSegmentIds().contains(segment.getId()));
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java
index 72fcec4ccb..991b0ec504 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java
@@ -20,8 +20,11 @@ package org.apache.kylin.job.dao;
import java.util.Map;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.MetadataConstants;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -74,4 +77,9 @@ public class JobStatistics extends JobStatisticsBasic {
jobStatisticsByModels.put(model, jobStatisticsByModel);
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + getProject() + ResourceStore.JOB_STATISTICS + "/" + date + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java
index c05743c3cf..c1b6f7902d 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java
@@ -20,6 +20,7 @@ package org.apache.kylin.job.dao;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
@@ -39,15 +40,22 @@ public class JobStatisticsBasic extends RootPersistentEntity {
@JsonProperty("total_byte_size")
private long totalByteSize;
+ @JsonIgnore
+ private String project;
+
public void update(long duration, long byteSize, int deltaCount) {
this.count += deltaCount;
this.totalDuration += duration;
this.totalByteSize += byteSize;
}
- public JobStatisticsBasic(long totalDuration, long totalByteSize) {
- this.count = 1;
+ public JobStatisticsBasic(int count, long totalDuration, long totalByteSize) {
+ this.count = count;
this.totalDuration = totalDuration;
this.totalByteSize = totalByteSize;
}
+
+ public JobStatisticsBasic(long totalDuration, long totalByteSize) {
+ this(1, totalDuration, totalByteSize);
+ }
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java
index 91acd80627..65d98d18c1 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java
@@ -35,12 +35,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
public class JobStatisticsManager {
public static JobStatisticsManager getInstance(KylinConfig config, String project) {
@@ -86,28 +85,37 @@ public class JobStatisticsManager {
return crud.listAll();
}
+ public JobStatistics copyForWrite(JobStatistics jobStatistics) {
+ if (jobStatistics.getProject() == null) {
+ jobStatistics.setProject(project);
+ }
+ return crud.copyForWrite(jobStatistics);
+ }
+
public JobStatistics updateStatistics(long date, String model, long duration, long byteSize, int deltaCount) {
JobStatistics jobStatistics = crud.get(String.valueOf(date));
- JobStatistics jobStatisticsToUpdate;
if (jobStatistics == null) {
- jobStatisticsToUpdate = new JobStatistics(date, model, duration, byteSize);
- return crud.save(jobStatisticsToUpdate);
+ jobStatistics = copyForWrite(new JobStatistics(date, model, duration, byteSize));
+ // double check whether jobStatistics already exists.
+ if (jobStatistics.getMvcc() == -1) {
+ return crud.save(jobStatistics);
+ }
}
- jobStatisticsToUpdate = crud.copyForWrite(jobStatistics);
+ JobStatistics jobStatisticsToUpdate = copyForWrite(jobStatistics);
jobStatisticsToUpdate.update(model, duration, byteSize, deltaCount);
return crud.save(jobStatisticsToUpdate);
}
public JobStatistics updateStatistics(long date, long duration, long byteSize, int deltaCount) {
JobStatistics jobStatistics = crud.get(String.valueOf(date));
- JobStatistics jobStatisticsToUpdate;
if (jobStatistics == null) {
- jobStatisticsToUpdate = new JobStatistics(date, duration, byteSize);
- return crud.save(jobStatisticsToUpdate);
+ // double check whether jobStatistics already exists.
+ jobStatistics = copyForWrite(new JobStatistics(date, duration, byteSize));
+ return crud.save(jobStatistics);
}
- jobStatisticsToUpdate = crud.copyForWrite(jobStatistics);
+ JobStatistics jobStatisticsToUpdate = copyForWrite(jobStatistics);
jobStatisticsToUpdate.update(duration, byteSize, deltaCount);
return crud.save(jobStatisticsToUpdate);
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java
index 130c5539a0..bcc4a497f2 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java
@@ -26,25 +26,20 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import lombok.val;
/**
*/
public class NExecutableDao {
- private static final Serializer<ExecutablePO> JOB_SERIALIZER = new JsonSerializer<>(ExecutablePO.class);
private static final Logger logger = LoggerFactory.getLogger(NExecutableDao.class);
public static NExecutableDao getInstance(KylinConfig config, String project) {
@@ -103,6 +98,13 @@ public class NExecutableDao {
return crud.get(uuid);
}
+ public ExecutablePO copyForWrite(ExecutablePO po) {
+ if (po.getProject() == null) {
+ po.setProject(project);
+ }
+ return crud.copyForWrite(po);
+ }
+
public ExecutablePO addJob(ExecutablePO job) {
if (getJobByUuid(job.getUuid()) != null) {
throw new IllegalArgumentException("job id:" + job.getUuid() + " already exists");
@@ -124,7 +126,7 @@ public class NExecutableDao {
public void updateJob(String uuid, Predicate<ExecutablePO> updater) {
val job = getJobByUuid(uuid);
Preconditions.checkNotNull(job);
- val copyForWrite = JsonUtil.copyBySerialization(job, JOB_SERIALIZER, null);
+ val copyForWrite = copyForWrite(job);
if (updater.test(copyForWrite)) {
crud.save(copyForWrite);
}
@@ -137,7 +139,7 @@ public class NExecutableDao {
} else {
ExecutablePO executablePOFromCache = getJobByUuid(uuid);
Preconditions.checkNotNull(executablePOFromCache);
- val copyForWrite = JsonUtil.copyBySerialization(executablePOFromCache, JOB_SERIALIZER, null);
+ val copyForWrite = copyForWrite(executablePOFromCache);
updating.put(uuid, copyForWrite);
executablePO = copyForWrite;
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
index 5eca8383e9..83ce486279 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
@@ -199,10 +199,11 @@ public class NExecutableManager {
}
public void addJob(ExecutablePO executablePO) {
- addJobOutput(executablePO);
- executableDao.addJob(executablePO);
+ ExecutablePO copy = executableDao.copyForWrite(executablePO);
+ addJobOutput(copy);
+ executableDao.addJob(copy);
- String jobType = executablePO.getJobType() == null ? "" : executablePO.getJobType().name();
+ String jobType = copy.getJobType() == null ? "" : copy.getJobType().name();
// dispatch job-created message out
if (KylinConfig.getInstanceFromEnv().isUTEnv()) {
EventBusFactory.getInstance().postAsync(new JobReadyNotifier(project));
@@ -1410,7 +1411,11 @@ public class NExecutableManager {
val thread = scheduler.getContext().getRunningJobThread(executable);
if (thread != null) {
logger.info("Interrupt Job [{}] thread and remove in ExecutableContext", executable.getDisplayName());
- thread.interrupt();
+ // workaround for https://olapio.atlassian.net/browse/KE-41836
+ if (Arrays.stream(thread.getStackTrace())
+ .noneMatch(stack -> stack.getClassName().contains("FetcherRunner"))) {
+ thread.interrupt();
+ }
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java
index 15fb55a597..52a8db9fee 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java
@@ -74,6 +74,10 @@ public class TempStatementManager {
public TempStatementEntity getTempStatEntity(String sessionId, String statementId) {
return crud.get(TempStatementEntity.resourceName(sessionId, statementId));
}
+
+ private TempStatementEntity copyForWrite(TempStatementEntity entity) {
+ return crud.copyForWrite(entity);
+ }
public void updateTempStatement(String statementId, String statement) throws IOException {
updateTempStatement(TempStatementEntity.DEFAULT_SESSION_ID, statementId, statement);
@@ -86,13 +90,13 @@ public class TempStatementManager {
}
private TempStatementEntity prepareToOverwrite(TempStatementEntity entity, TempStatementEntity origin) {
+ TempStatementEntity copy = copyForWrite(entity);
if (origin == null) {
// create
- entity.updateRandomUuid();
+ copy.updateRandomUuid();
} else {
// update
- entity.setUuid(origin.getUuid());
- entity.setLastModified(origin.getLastModified());
+ copy.setStatement(entity.getStatement());
}
return entity;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java
index d15b9f741d..0b51b10574 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java
@@ -38,6 +38,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -57,9 +58,13 @@ public class AclTCR extends RootPersistentEntity {
//wrap read only aclTCR
private String resourceName;
+ private String project;
+ private boolean principal;
- public void init(String resourceName) {
+ public void init(String resourceName, String project, boolean principal) {
this.resourceName = resourceName;
+ this.project = project;
+ this.principal = principal;
}
@JsonProperty
@@ -70,6 +75,16 @@ public class AclTCR extends RootPersistentEntity {
return resourceName;
}
+ @Override
+ public String getResourcePath() {
+ return "/" + project + "/acl/" + (principal ? "user/" : "group/") + resourceName()
+ + MetadataConstants.FILE_SURFIX;
+ }
+
+ public boolean isPrincipal() {
+ return principal;
+ }
+
public Table getTable() {
return table;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
index 044bdfa7ff..7280ae81ec 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
@@ -89,7 +89,7 @@ public class AclTCRManager {
AclTCR.class) {
@Override
protected AclTCR initEntityAfterReload(AclTCR acl, String resourceName) {
- acl.init(resourceName);
+ acl.init(resourceName, project, true);
return acl;
}
};
@@ -99,7 +99,7 @@ public class AclTCRManager {
AclTCR.class) {
@Override
protected AclTCR initEntityAfterReload(AclTCR acl, String resourceName) {
- acl.init(resourceName);
+ acl.init(resourceName, project, false);
return acl;
}
};
@@ -134,21 +134,18 @@ public class AclTCRManager {
}
public void updateAclTCR(AclTCR updateTo, String sid, boolean principal) {
- updateTo.init(sid);
+ updateTo.init(sid, project, principal);
if (principal) {
- doUpdate(updateTo, sid, userCrud);
+ doUpdate(updateTo, userCrud);
} else {
- doUpdate(updateTo, sid, groupCrud);
+ doUpdate(updateTo, groupCrud);
}
}
- private void doUpdate(AclTCR updateTo, String sid, CachedCrudAssist<AclTCR> crud) {
- AclTCR copied;
- AclTCR origin = crud.get(sid);
- if (Objects.isNull(origin)) {
- copied = updateTo;
- } else {
- copied = crud.copyForWrite(origin);
+ private void doUpdate(AclTCR updateTo, CachedCrudAssist<AclTCR> crud) {
+ AclTCR cached = crud.get(updateTo.resourceName());
+ AclTCR copied = crud.copyForWrite(cached == null ? updateTo : cached);
+ if (copied.getMvcc() != -1) {
copied.setTable(updateTo.getTable());
}
crud.save(copied);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java
index 47b5f076d9..a5a114bc2c 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java
@@ -20,7 +20,7 @@ package org.apache.kylin.metadata.cachesync;
import org.apache.kylin.common.persistence.MissingRootPersistentEntity;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-
+import org.apache.kylin.common.persistence.metadata.FileMetadataStore;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.AllArgsConstructor;
@@ -53,6 +53,10 @@ public class CacheReloadChecker<T extends RootPersistentEntity> {
if (raw.getMvcc() != entity.getMvcc()) {
return true;
}
+ if ((store.getMetadataStore() instanceof FileMetadataStore)
+ && (raw.getTimestamp() != entity.getLastModified())) {
+ return true;
+ }
Preconditions.checkState(!(entity instanceof MissingRootPersistentEntity));
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
index 3793ca7ff2..fdc4851213 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
@@ -21,24 +21,27 @@ package org.apache.kylin.metadata.cachesync;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.ThreadUtil;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.util.BrokenEntityProxy;
-
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.cache.Cache;
import org.apache.kylin.guava30.shaded.common.cache.CustomKeyEquivalenceCacheBuilder;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.util.BrokenEntityProxy;
import lombok.AccessLevel;
import lombok.Getter;
@@ -88,9 +91,47 @@ public abstract class CachedCrudAssist<T extends RootPersistentEntity> {
this.checkCopyOnWrite = check;
}
- // Make copy of an entity such that update can apply on the copy.
- // Note cached and shared object MUST NOT be updated directly.
+ // If this is a new entity or the reload entity is broken, return a copy of this entity.
+ // If the reload entity is null, just return null.
+ // Otherwise return a copy of the reload entity.
public T copyForWrite(T entity) {
+ if (entity == null) {
+ return null;
+ }
+ String resourcePath = resourcePath(entity.resourceName());
+ return MemoryLockUtils.doWithLock(entity, resourcePath, false, store, () -> {
+ if (entity.getMvcc() == -1) {
+ return copyIfCachedAndShared(entity);
+ }
+ T reloadedEntity = get(entity.resourceName());
+ if (reloadedEntity == null) {
+ return null;
+ } else if (reloadedEntity.isBroken()) {
+ return copyIfCachedAndShared(entity);
+ } else {
+ return copyIfCachedAndShared(reloadedEntity);
+ }
+ });
+ }
+
+ // Used by managers which doesn't have cache.
+ public static <T extends RootPersistentEntity> T copyForWrite(T entity, Serializer<T> serializer,
+ @Nullable BiConsumer<T, String> initEntity, ResourceStore store) {
+ String resourcePath = entity.getResourcePath();
+ return MemoryLockUtils.doWithLock(entity, resourcePath, false, store, () -> {
+ T reloadedEntity = store.getResource(entity.getResourcePath(), serializer);
+ if (reloadedEntity == null || reloadedEntity.isBroken()) {
+ return JsonUtil.copyForWrite(entity, serializer, initEntity);
+ } else {
+ if (initEntity != null) {
+ initEntity.accept(reloadedEntity, reloadedEntity.resourceName());
+ }
+ return JsonUtil.copyForWrite(reloadedEntity, serializer, initEntity);
+ }
+ });
+ }
+
+ public T copyIfCachedAndShared(T entity) {
return JsonUtil.copyForWrite(entity, serializer, this::initEntityAfterReload);
}
@@ -98,7 +139,7 @@ public abstract class CachedCrudAssist<T extends RootPersistentEntity> {
return JsonUtil.copyBySerialization(entity, serializer, this::initEntityAfterReload);
}
- String resourcePath(String resourceName) {
+ public String resourcePath(String resourceName) {
Preconditions.checkArgument(StringUtils.isNotEmpty(resourceName),
"The resource name \"{}\" cannot contain white character", resourceName);
return resRootPath + "/" + resourceName + resPathSuffix;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
index ee80805d1a..83edf8385b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
@@ -365,6 +365,11 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
.append("/").append(name).append(MetadataConstants.FILE_SURFIX).toString();
}
+ @Override
+ public List<String> getLockPaths(String ignored) {
+ return getModel().getLockPaths();
+ }
+
public String getProject() {
return project;
}
@@ -583,6 +588,9 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
public void setRuleBasedIndex(RuleBasedIndex ruleBasedIndex, Set<LayoutEntity> reloadLayouts, boolean reuseStartId,
boolean markToBeDeleted, boolean restoreDeletedIndex) {
+ if (ruleBasedIndex == null) {
+ return;
+ }
checkIsNotCachedAndShared();
ruleBasedIndex.adjustMeasures();
if (CollectionUtils.isEmpty(ruleBasedIndex.getMeasures())) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
index c9d1ea969e..9f86786f11 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
@@ -26,23 +26,22 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.ManagementType;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.model.ManagementType;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.var;
@@ -113,61 +112,74 @@ public class NDataLoadingRangeManager {
}
public NDataLoadingRange createDataLoadingRange(NDataLoadingRange dataLoadingRange) {
- checkNDataLoadingRangeIdentify(dataLoadingRange);
- checkNDataLoadingRangeExist(dataLoadingRange);
+ NDataLoadingRange copy = copyForWrite(dataLoadingRange);
+ checkNDataLoadingRangeIdentify(copy);
+ checkNDataLoadingRangeExist(copy);
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(config, project);
- String tableName = dataLoadingRange.getTableName();
+ String tableName = copy.getTableName();
TableDesc tableDesc = tableMetadataManager.getTableDesc(tableName);
if (tableDesc == null) {
- throw new IllegalArgumentException(DATA_LOADING_RANGE + dataLoadingRange.resourceName() + "' 's table "
- + tableName + " does not exists");
+ throw new IllegalArgumentException(
+ DATA_LOADING_RANGE + copy.resourceName() + "' 's table " + tableName + " does not exists");
}
- String columnName = dataLoadingRange.getColumnName();
+ String columnName = copy.getColumnName();
ColumnDesc columnDesc = tableDesc.findColumnByName(columnName);
if (columnDesc == null) {
- throw new IllegalArgumentException(DATA_LOADING_RANGE + dataLoadingRange.resourceName() + "' 's column "
- + columnName + " does not exists");
+ throw new IllegalArgumentException(
+ DATA_LOADING_RANGE + copy.resourceName() + "' 's column " + columnName + " does not exists");
}
String columnType = columnDesc.getDatatype();
DataType dataType = DataType.getType(columnType);
if (dataType == null || !dataType.isLegalPartitionColumnType()) {
- throw new IllegalArgumentException(DATA_LOADING_RANGE + dataLoadingRange.resourceName() + "' 's column "
- + columnName + " 's dataType does not support partition column");
+ throw new IllegalArgumentException(DATA_LOADING_RANGE + copy.resourceName() + "' 's column " + columnName
+ + " 's dataType does not support partition column");
}
- return crud.save(dataLoadingRange);
+ return crud.save(copy);
}
public NDataLoadingRange appendSegmentRange(NDataLoadingRange dataLoadingRange, SegmentRange segmentRange) {
- NDataLoadingRange copyForWrite = copyForWrite(dataLoadingRange);
- val coveredRange = copyForWrite.getCoveredRange();
- if (coveredRange == null) {
- copyForWrite.setCoveredRange(segmentRange);
- } else {
- if (coveredRange.connects(segmentRange)) {
- copyForWrite.setCoveredRange(coveredRange.coverWith(segmentRange));
- } else if (segmentRange.connects(coveredRange)) {
- copyForWrite.setCoveredRange(segmentRange.coverWith(coveredRange));
+ return updateDataLoadingRange(dataLoadingRange.getTableName(), copyForWrite -> {
+ val coveredRange = copyForWrite.getCoveredRange();
+ if (coveredRange == null) {
+ copyForWrite.setCoveredRange(segmentRange);
} else {
- throw new IllegalArgumentException("NDataLoadingRange appendSegmentRange " + segmentRange
- + " has overlaps/gap with existing segmentRanges " + copyForWrite.getCoveredRange());
+ if (coveredRange.connects(segmentRange)) {
+ copyForWrite.setCoveredRange(coveredRange.coverWith(segmentRange));
+ } else if (segmentRange.connects(coveredRange)) {
+ copyForWrite.setCoveredRange(segmentRange.coverWith(coveredRange));
+ } else {
+ throw new IllegalArgumentException("NDataLoadingRange appendSegmentRange " + segmentRange
+ + " has overlaps/gap with existing segmentRanges " + copyForWrite.getCoveredRange());
+ }
}
- }
- return updateDataLoadingRange(copyForWrite);
+ return true;
+ });
}
+ /**
+ * @deprecated Use updateDataLoadingRange(String tableName, DataLoadingRangeUpdater updater) instead.
+ */
+ @Deprecated
public NDataLoadingRange updateDataLoadingRange(NDataLoadingRange dataLoadingRange) {
- if (getStore().getConfig().isCheckCopyOnWrite()) {
- if (dataLoadingRange.isCachedAndShared())
- throw new IllegalStateException();
- }
- checkNDataLoadingRangeIdentify(dataLoadingRange);
- checkNDataLoadingRangeNotExist(dataLoadingRange);
-
- return crud.save(dataLoadingRange);
+ return updateDataLoadingRange(dataLoadingRange.getTableName(), copyForWrite -> {
+ dataLoadingRange.copyPropertiesTo(copyForWrite);
+ return true;
+ });
}
+ public NDataLoadingRange updateDataLoadingRange(String tableName, DataLoadingRangeUpdater updater) {
+ NDataLoadingRange cached = getDataLoadingRange(tableName);
+ NDataLoadingRange copy = copyForWrite(cached);
+ if (!updater.modify(copy)) {
+ return null;
+ }
+ checkNDataLoadingRangeIdentify(copy);
+ checkNDataLoadingRangeNotExist(copy);
+ return crud.save(copy);
+ }
+
public NDataLoadingRange copyForWrite(NDataLoadingRange dataLoadingRange) {
return crud.copyForWrite(dataLoadingRange);
}
@@ -251,16 +263,21 @@ public class NDataLoadingRangeManager {
if (loadingRange == null) {
return;
} else {
- val copy = copyForWrite(loadingRange);
- val coveredRange = copy.getCoveredRange();
- if (coveredRange == null) {
- return;
- }
- if (lastSegment.getSegRange().overlaps(coveredRange)) {
- copy.setCoveredRange(lastSegment.getSegRange().getEndDeviation(coveredRange));
- updateDataLoadingRange(copy);
- }
-
+ updateDataLoadingRange(loadingRange.getTableName(), copyForWrite -> {
+ val coveredRange = copyForWrite.getCoveredRange();
+ if (coveredRange == null) {
+ return false;
+ }
+ if (lastSegment.getSegRange().overlaps(coveredRange)) {
+ copyForWrite.setCoveredRange(lastSegment.getSegRange().getEndDeviation(coveredRange));
+ return true;
+ }
+ return false;
+ });
}
}
+
+ public interface DataLoadingRangeUpdater {
+ boolean modify(NDataLoadingRange copyForWrite);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java
index 86676b7f8b..982bb5d7ff 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java
@@ -31,13 +31,13 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import lombok.val;
/**
@@ -65,6 +65,7 @@ class NDataSegDetailsManager {
private KylinConfig kylinConfig;
private String project;
+ private ResourceStore resourceStore;
private NDataSegDetailsManager(KylinConfig config, String project) {
if (!UnitOfWork.isAlreadyInTransaction())
@@ -72,6 +73,7 @@ class NDataSegDetailsManager {
System.identityHashCode(config), project);
this.kylinConfig = config;
this.project = project;
+ this.resourceStore = ResourceStore.getKylinMetaStore(this.kylinConfig);
}
public KylinConfig getConfig() {
@@ -79,9 +81,18 @@ class NDataSegDetailsManager {
}
private ResourceStore getStore() {
- return ResourceStore.getKylinMetaStore(this.kylinConfig);
+ return resourceStore;
}
+ private NDataSegDetails initEntity(NDataSegDetails dataSegDetails, String resourceName) {
+ dataSegDetails.setProject(project);
+ return dataSegDetails;
+ }
+
+ public NDataSegDetails copyForWrite(NDataSegDetails details) {
+ return CachedCrudAssist.copyForWrite(details, DATA_SEG_LAYOUT_INSTANCES_SERIALIZER, this::initEntity, resourceStore);
+ }
+
NDataSegDetails getForSegment(NDataflow df, String segId) {
NDataSegDetails instances = getStore().getResource(getResourcePathForSegment(df.getUuid(), segId),
DATA_SEG_LAYOUT_INSTANCES_SERIALIZER);
@@ -127,20 +138,21 @@ class NDataSegDetailsManager {
NDataSegDetails details = getForSegment(df, segId);
if (details == null)
details = NDataSegDetails.newSegDetails(df, segId);
+ NDataSegDetails copy = copyForWrite(details);
if (toUpsert.containsKey(segId)) {
for (NDataLayout c : toUpsert.get(segId)) {
- c.setSegDetails(details);
- details.addLayout(c);
+ c.setSegDetails(copy);
+ copy.addLayout(c);
}
}
if (toRemove.containsKey(segId)) {
for (NDataLayout c : toRemove.get(segId)) {
- details.removeLayout(c);
+ copy.removeLayout(c);
}
}
- upsertForSegmentQuietly(details);
+ upsertForSegmentQuietly(copy);
}
if (update.getToRemoveSegs() != null) {
@@ -163,13 +175,24 @@ class NDataSegDetailsManager {
}
}
- NDataSegDetails upsertForSegment(NDataSegDetails details) {
+ private NDataSegDetails upsertForSegment(NDataSegDetails details) {
Preconditions.checkNotNull(details, "NDataSegDetails cannot be null.");
getStore().checkAndPutResource(details.getResourcePath(), details, DATA_SEG_LAYOUT_INSTANCES_SERIALIZER);
return details;
}
+ public NDataSegDetails updateDetails(NDataSegment seg, NDataSegDetailsUpdater updater) {
+ NDataSegDetails details = copyForWrite(getForSegment(seg));
+ updater.modify(details);
+ upsertForSegment(details);
+ return getForSegment(seg);
+ }
+
+ interface NDataSegDetailsUpdater {
+ void modify(NDataSegDetails copyForWrite);
+ }
+
private void removeForSegmentQuietly(NDataflow df, String segId) {
try {
removeForSegment(df, segId);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
index 022d7a39c4..ce2d5c817e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
@@ -31,6 +31,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -48,6 +49,9 @@ import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.model.ManagementType;
import org.apache.kylin.metadata.model.NDataModel;
@@ -66,10 +70,6 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.var;
@@ -277,14 +277,15 @@ public class NDataflowManager implements IRealizationProvider {
public NDataflow createDataflow(IndexPlan plan, String owner, RealizationStatusEnum realizationStatusEnum) {
NDataflow df = NDataflow.create(plan, realizationStatusEnum);
df.initAfterReload((KylinConfigExt) plan.getConfig(), project);
+ NDataflow copy = copyForWrite(df);
// save dataflow
- df.getSegments().validate();
- crud.save(df);
+ copy.getSegments().validate();
+ crud.save(copy);
- fillDf(df);
+ fillDf(copy);
- return df;
+ return copy;
}
public void fillDf(NDataflow df) {
@@ -339,22 +340,22 @@ public class NDataflowManager implements IRealizationProvider {
}
public NDataSegment appendPartitions(String dfId, String segId, List<String[]> partitionValues) {
- val copy = copy(getDataflow(dfId));
- val segmentCopy = copy.getSegment(segId);
- partitionValues.forEach(partitionValue -> {
- if (copy.getSegment(segmentCopy.getId()).isPartitionOverlap(partitionValue)) {
- throw new IllegalArgumentException(
- String.format(Locale.ROOT, "Duplicate partition value [%s] found in segment [%s]",
- Arrays.toString(partitionValue), segmentCopy.getId()));
- }
- });
- val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- Set<Long> addPartitions = modelManager.addPartitionsIfAbsent(copy.getModel(), partitionValues);
- addPartitions.forEach(partition -> {
- segmentCopy.getMultiPartitions().add(new SegmentPartition(partition));
+ updateDataflow(dfId, copyForWrite -> {
+ val segmentCopy = copyForWrite.getSegment(segId);
+ partitionValues.forEach(partitionValue -> {
+ if (copyForWrite.getSegment(segmentCopy.getId()).isPartitionOverlap(partitionValue)) {
+ throw new IllegalArgumentException(
+ String.format(Locale.ROOT, "Duplicate partition value [%s] found in segment [%s]",
+ Arrays.toString(partitionValue), segmentCopy.getId()));
+ }
+ });
+ val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ Set<Long> addPartitions = modelManager.addPartitionsIfAbsent(copyForWrite.getModel(), partitionValues);
+ addPartitions.forEach(partition -> {
+ segmentCopy.getMultiPartitions().add(new SegmentPartition(partition));
+ });
});
- crud.save(copy);
- return segmentCopy;
+ return copy(getDataflow(dfId)).getSegment(segId);
}
public NDataSegment appendSegmentForStreaming(NDataflow df, SegmentRange segRange) {
@@ -581,6 +582,10 @@ public class NDataflowManager implements IRealizationProvider {
return toRemoveSegs;
}
+ public NDataflow copyForWrite(NDataflow df) {
+ return crud.copyForWrite(df);
+ }
+
public NDataflow copy(NDataflow df) {
return crud.copyBySerialization(df);
}
@@ -632,7 +637,7 @@ public class NDataflowManager implements IRealizationProvider {
*/
public NDataflow updateDataflow(String dfId, NDataflowUpdater updater) {
NDataflow cached = getDataflow(dfId);
- NDataflow copy = copy(cached);
+ NDataflow copy = copyForWrite(cached);
updater.modify(copy);
Set<String> copySegIdSet = copy.getSegments().stream().map(NDataSegment::getId).collect(Collectors.toSet());
@@ -661,15 +666,23 @@ public class NDataflowManager implements IRealizationProvider {
return getDataflow(modelId).getLastBuildTime();
}
- public void updateDataflowDetailsLayouts(final NDataSegment seg, final List<NDataLayout> layouts) {
+ public void updateDataflowDetailsLayouts(final NDataSegment seg, final List<Long> toRemoveLayouts,
+ final List<Long> toAddLayouts) {
NDataSegDetailsManager segDetailsManager = NDataSegDetailsManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- NDataSegDetails details = segDetailsManager.getForSegment(seg);
- details.setLayouts(layouts);
- NDataSegDetailsManager.getInstance(KylinConfig.getInstanceFromEnv(), project).upsertForSegment(details);
+ segDetailsManager.updateDetails(seg, copyForWrite -> {
+ List<NDataLayout> layouts = new LinkedList<>(copyForWrite.getAllLayouts());
+ layouts.removeIf(layout -> toRemoveLayouts.contains(layout.getLayoutId()));
+ List<Long> existLayouts = layouts.stream().map(NDataLayout::getLayoutId).collect(Collectors.toList());
+ for (Long layoutId : toAddLayouts) {
+ if (!existLayouts.contains(layoutId)) {
+ layouts.add(NDataLayout.newDataLayout(copyForWrite, layoutId));
+ }
+ }
+ copyForWrite.setLayouts(layouts);
+ });
updateDataflow(seg.getDataflow().getId(),
copyForWrite -> updateSegmentStatus(copyForWrite.getSegment(seg.getId())));
-
}
public NDataflow updateDataflow(final NDataflowUpdate update) {
@@ -803,20 +816,20 @@ public class NDataflowManager implements IRealizationProvider {
}
public void removeSegmentPartition(String dfId, Set<Long> toBeDeletedPartIds, Set<String> segments) {
- val dfCopy = copy(getDataflow(dfId));
- val updateSegments = new Segments<NDataSegment>();
- if (CollectionUtils.isEmpty(segments)) {
- updateSegments.addAll(dfCopy.getSegments());
- } else {
- dfCopy.getSegments().forEach(segment -> {
- if (segments.contains(segment.getId())) {
- updateSegments.add(segment);
- }
- });
- }
- updateSegments.forEach(segment -> segment.getMultiPartitions()
- .removeIf(partition -> toBeDeletedPartIds.contains(partition.getPartitionId())));
- crud.save(dfCopy);
+ updateDataflow(dfId, copyForWrite -> {
+ val updateSegments = new Segments<NDataSegment>();
+ if (CollectionUtils.isEmpty(segments)) {
+ updateSegments.addAll(copyForWrite.getSegments());
+ } else {
+ copyForWrite.getSegments().forEach(segment -> {
+ if (segments.contains(segment.getId())) {
+ updateSegments.add(segment);
+ }
+ });
+ }
+ updateSegments.forEach(segment -> segment.getMultiPartitions()
+ .removeIf(partition -> toBeDeletedPartIds.contains(partition.getPartitionId())));
+ });
}
public void removeLayoutPartition(String dfId, Set<Long> toBeDeletedPartIds, Set<String> segments) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
index e5e605da6e..670e8f5af7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
@@ -32,9 +32,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.metadata.cube.cuboid.CuboidScheduler;
import org.apache.kylin.metadata.cube.model.validation.NIndexPlanValidator;
import org.apache.kylin.metadata.model.NDataModel;
@@ -42,9 +44,6 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.val;
public class NIndexPlanManager {
@@ -94,6 +93,13 @@ public class NIndexPlanManager {
this.crud.setCheckCopyOnWrite(true);
}
+ public IndexPlan copyForWrite(IndexPlan plan) {
+ if (plan.getProject() == null) {
+ plan.setProject(project);
+ }
+ return crud.copyForWrite(plan);
+ }
+
public IndexPlan copy(IndexPlan plan) {
return crud.copyBySerialization(plan);
}
@@ -123,8 +129,6 @@ public class NIndexPlanManager {
public IndexPlan createIndexPlan(IndexPlan indexPlan) {
if (indexPlan.getUuid() == null)
throw new IllegalArgumentException();
- if (crud.contains(indexPlan.getUuid()))
- throw new IllegalArgumentException("IndexPlan '" + indexPlan.getUuid() + "' already exists");
try {
// init the cube plan if not yet
@@ -135,18 +139,22 @@ public class NIndexPlanManager {
indexPlan.addError(e.getMessage());
}
+ IndexPlan copy = copyForWrite(indexPlan);
+ if (crud.contains(copy.getUuid()))
+ throw new IllegalArgumentException("IndexPlan '" + indexPlan.getUuid() + "' already exists");
+
// Check base validation
- if (!indexPlan.getError().isEmpty()) {
- throw new IllegalArgumentException(indexPlan.getErrorMsg());
+ if (!copy.getError().isEmpty()) {
+ throw new IllegalArgumentException(copy.getErrorMsg());
}
// Semantic validation
NIndexPlanValidator validator = new NIndexPlanValidator();
- ValidateContext context = validator.validate(indexPlan);
+ ValidateContext context = validator.validate(copy);
if (!context.ifPass()) {
- throw new IllegalArgumentException(indexPlan.getErrorMsg());
+ throw new IllegalArgumentException(copy.getErrorMsg());
}
- return save(indexPlan);
+ return save(copy);
}
public interface NIndexPlanUpdater {
@@ -155,14 +163,18 @@ public class NIndexPlanManager {
public IndexPlan updateIndexPlan(String indexPlanId, NIndexPlanUpdater updater) {
IndexPlan cached = getIndexPlan(indexPlanId);
- IndexPlan copy = copy(cached);
+ IndexPlan copy = copyForWrite(cached);
updater.modify(copy);
- return updateIndexPlan(copy);
+ return innerUpdateIndexPlan(copy);
}
// use the NIndexPlanUpdater instead
@Deprecated
public IndexPlan updateIndexPlan(IndexPlan indexPlan) {
+ return updateIndexPlan(indexPlan.getUuid(), indexPlan::copyPropertiesTo);
+ }
+
+ private IndexPlan innerUpdateIndexPlan(IndexPlan indexPlan) {
if (indexPlan.isCachedAndShared())
throw new IllegalStateException();
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
index 1bbd33831f..4b08f4a5cc 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
@@ -18,8 +18,11 @@
package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.MetadataConstants;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.EqualsAndHashCode;
@@ -36,7 +39,15 @@ public class AbstractAsyncTask extends RootPersistentEntity {
@JsonProperty("task_type")
private String taskType;
+ @JsonIgnore
+ private String project;
+
public AbstractAsyncTask(String taskType) {
this.taskType = taskType;
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + project + ResourceStore.ASYNC_TASK + "/" + uuid + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java
index 55eb565364..7560659ab8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java
@@ -20,13 +20,14 @@ package org.apache.kylin.metadata.favorite;
import java.util.Map;
+import org.apache.kylin.common.annotation.Clarification;
+
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
-import org.apache.kylin.common.annotation.Clarification;
@Getter
@Setter
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java
index d2a4daa9b5..afdbe3eb61 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java
@@ -33,6 +33,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import lombok.extern.slf4j.Slf4j;
@@ -47,11 +48,13 @@ public class AsyncTaskManager {
private final KylinConfig kylinConfig;
private final ResourceStore resourceStore;
private final String resourceRoot;
+ private final String project;
private AsyncTaskManager(KylinConfig kylinConfig, String project) {
if (!UnitOfWork.isAlreadyInTransaction())
log.info("Initializing AccelerateTagManager with KylinConfig Id: {} for project {}",
System.identityHashCode(kylinConfig), project);
+ this.project = project;
this.kylinConfig = kylinConfig;
resourceStore = ResourceStore.getKylinMetaStore(this.kylinConfig);
this.resourceRoot = "/" + project + ResourceStore.ASYNC_TASK;
@@ -76,6 +79,13 @@ public class AsyncTaskManager {
}
}
+ public AsyncAccelerationTask copyForWrite(AsyncAccelerationTask task) {
+ if (task.getProject() == null) {
+ task.setProject(project);
+ }
+ return CachedCrudAssist.copyForWrite(task, ASYNC_ACCELERATION_SERIALIZER, null, resourceStore);
+ }
+
public AbstractAsyncTask get(String taskType) {
List<AsyncAccelerationTask> asyncAccelerationTaskList = Lists.newArrayList();
if (taskType.equalsIgnoreCase(ASYNC_ACCELERATION_TASK)) {
@@ -90,11 +100,12 @@ public class AsyncTaskManager {
public static void resetAccelerationTagMap(String project) {
log.info("reset acceleration tag for project({})", project);
EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) getInstance(
- KylinConfig.getInstanceFromEnv(), project).get(ASYNC_ACCELERATION_TASK);
- asyncAcceleration.setAlreadyRunning(false);
- asyncAcceleration.setUserRefreshedTagMap(Maps.newHashMap());
- getInstance(KylinConfig.getInstanceFromEnv(), project).save(asyncAcceleration);
+ AsyncTaskManager manager = getInstance(KylinConfig.getInstanceFromEnv(), project);
+ AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) manager.get(ASYNC_ACCELERATION_TASK);
+ AsyncAccelerationTask copied = manager.copyForWrite(asyncAcceleration);
+ copied.setAlreadyRunning(false);
+ copied.setUserRefreshedTagMap(Maps.newHashMap());
+ getInstance(KylinConfig.getInstanceFromEnv(), project).save(copied);
return null;
}, project);
log.info("rest acceleration tag successfully for project({})", project);
@@ -114,10 +125,11 @@ public class AsyncTaskManager {
log.info("start to clean acceleration tag by user");
EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) getInstance(
- KylinConfig.getInstanceFromEnv(), project).get(ASYNC_ACCELERATION_TASK);
- asyncAcceleration.getUserRefreshedTagMap().put(userName, false);
- getInstance(KylinConfig.getInstanceFromEnv(), project).save(asyncAcceleration);
+ AsyncTaskManager manager = getInstance(KylinConfig.getInstanceFromEnv(), project);
+ AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) manager.get(ASYNC_ACCELERATION_TASK);
+ AsyncAccelerationTask copied = manager.copyForWrite(asyncAcceleration);
+ copied.getUserRefreshedTagMap().put(userName, false);
+ getInstance(KylinConfig.getInstanceFromEnv(), project).save(copied);
return null;
}, project);
log.info("clean acceleration tag successfully for project({}: by user {})", project, userName);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java
index 89a3368182..f806795547 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java
@@ -23,14 +23,17 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.kylin.common.annotation.Clarification;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.RandomUtil;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.MetadataConstants;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -75,6 +78,9 @@ public class FavoriteRule extends RootPersistentEntity {
@JsonProperty("enabled")
private boolean enabled;
+ @JsonIgnore
+ private String project;
+
public static List<FavoriteRule> getAllDefaultRule() {
return FAVORITE_RULE_NAMES.stream().map(ruleName -> getDefaultRuleIfNull(null, ruleName))
.collect(Collectors.toList());
@@ -124,6 +130,11 @@ public class FavoriteRule extends RootPersistentEntity {
return null;
}
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + project + ResourceStore.QUERY_FILTER_RULE_RESOURCE_ROOT + "/" + resourceName() + MetadataConstants.FILE_SURFIX;
+ }
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
@NoArgsConstructor
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java
index ecbaa4bd80..fb685a2022 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java
@@ -29,14 +29,13 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.annotation.Clarification;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import lombok.val;
@Clarification(priority = Clarification.Priority.MAJOR, msg = "Enterprise")
@@ -75,6 +74,7 @@ public class FavoriteRuleManager {
this.crud = new CachedCrudAssist<FavoriteRule>(store, resourceRoot, FavoriteRule.class) {
@Override
protected FavoriteRule initEntityAfterReload(FavoriteRule entity, String resourceName) {
+ entity.setProject(project);
return entity;
}
};
@@ -112,6 +112,13 @@ public class FavoriteRuleManager {
return FavoriteRule.getDefaultRuleIfNull(getByName(ruleName), ruleName);
}
+ private FavoriteRule copyForWrite(FavoriteRule rule) {
+ if (rule.getProject() == null) {
+ rule.setProject(project);
+ }
+ return crud.copyForWrite(rule);
+ }
+
public void resetRule() {
FavoriteRule.getAllDefaultRule().forEach(this::updateRule);
}
@@ -121,7 +128,7 @@ public class FavoriteRuleManager {
}
public void updateRule(List<FavoriteRule.AbstractCondition> conditions, boolean isEnabled, String ruleName) {
- FavoriteRule copy = crud.copyForWrite(getOrDefaultByName(ruleName));
+ FavoriteRule copy = copyForWrite(getOrDefaultByName(ruleName));
copy.setEnabled(isEnabled);
@@ -140,10 +147,11 @@ public class FavoriteRuleManager {
@VisibleForTesting
public void createRule(final FavoriteRule rule) {
- if (getByName(rule.getName()) != null)
+ FavoriteRule copy = copyForWrite(rule);
+ if (getByName(copy.getName()) != null)
return;
- crud.save(rule);
+ crud.save(copy);
}
@VisibleForTesting
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
index 86eb29c09f..f5b105b207 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
@@ -18,6 +18,8 @@
package org.apache.kylin.metadata.favorite;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -26,6 +28,7 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import org.apache.kylin.metadata.MetadataConstants;
@Getter
@Setter
@@ -37,8 +40,16 @@ public class QueryHistoryIdOffset extends RootPersistentEntity {
@JsonProperty("query_history_stat_id_offset")
private long statMetaUpdateOffset;
+
+ @JsonIgnore
+ private String project;
public QueryHistoryIdOffset(long offset) {
this.offset = offset;
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + getProject() + ResourceStore.QUERY_HISTORY_ID_OFFSET + "/" + uuid + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java
index a96d43afd7..03fa25295b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java
@@ -24,8 +24,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,12 +40,14 @@ public class QueryHistoryIdOffsetManager {
private final KylinConfig kylinConfig;
private ResourceStore resourceStore;
private String resourceRoot;
+ private String project;
private QueryHistoryIdOffsetManager(KylinConfig kylinConfig, String project) {
if (!UnitOfWork.isAlreadyInTransaction())
logger.info("Initializing QueryHistoryIdOffsetManager with KylinConfig Id: {} for project {}",
System.identityHashCode(kylinConfig), project);
this.kylinConfig = kylinConfig;
+ this.project = project;
resourceStore = ResourceStore.getKylinMetaStore(this.kylinConfig);
this.resourceRoot = "/" + project + ResourceStore.QUERY_HISTORY_ID_OFFSET;
}
@@ -62,9 +65,27 @@ public class QueryHistoryIdOffsetManager {
return this.resourceRoot + "/" + uuid + MetadataConstants.FILE_SURFIX;
}
- public void save(QueryHistoryIdOffset idOffset) {
+ public QueryHistoryIdOffset copyForWrite(QueryHistoryIdOffset idOffset) {
+ if (idOffset.getProject() == null) {
+ idOffset.setProject(project);
+ }
+ return CachedCrudAssist.copyForWrite(idOffset, QUERY_HISTORY_ID_OFFSET_SERIALIZER, null, resourceStore);
+ }
+
+ private void save(QueryHistoryIdOffset idOffset) {
resourceStore.checkAndPutResource(path(idOffset.getUuid()), idOffset, QUERY_HISTORY_ID_OFFSET_SERIALIZER);
}
+
+ public void updateOffset(QueryHistoryIdOffsetUpdater updater) {
+ QueryHistoryIdOffset cached = get();
+ QueryHistoryIdOffset copy = copyForWrite(cached);
+ updater.modify(copy);
+ save(copy);
+ }
+
+ public interface QueryHistoryIdOffsetUpdater {
+ void modify(QueryHistoryIdOffset offset);
+ }
public QueryHistoryIdOffset get() {
List<QueryHistoryIdOffset> queryHistoryIdOffsetList = resourceStore.getAllResources(resourceRoot,
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java
index 1a466ff02c..690c1f7cdf 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java
@@ -76,22 +76,38 @@ public class JarInfoManager {
return String.format(Locale.ROOT, "%s_%s", jarTypeEnum, jarName);
}
+ private JarInfo copyForWrite(JarInfo jarInfo) {
+ return crud.copyForWrite(jarInfo);
+ }
+
public JarInfo createJarInfo(JarInfo jarInfo) {
if (Objects.isNull(jarInfo) || StringUtils.isEmpty(jarInfo.resourceName())) {
throw new IllegalArgumentException("jar info is null or resourceName is null");
}
- if (crud.contains(jarInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_JAR, jarInfo.getJarName());
+ JarInfo copy = copyForWrite(jarInfo);
+ if (crud.contains(copy.resourceName())) {
+ throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_JAR, copy.getJarName());
}
- jarInfo.updateRandomUuid();
- return crud.save(jarInfo);
+ copy.updateRandomUuid();
+ return crud.save(copy);
}
+ /**
+ * @deprecated Use updateJarInfo(JarTypeEnum jarTypeEnum, String jarName, JarInfoUpdater updater) instead.
+ */
+ @Deprecated
public JarInfo updateJarInfo(JarInfo jarInfo) {
- if (!crud.contains(jarInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_JAR, jarInfo.getJarName());
+ return updateJarInfo(jarInfo.getJarType(), jarInfo.getJarName(), jarInfo::copyPropertiesTo);
+ }
+
+ public JarInfo updateJarInfo(JarTypeEnum jarTypeEnum, String jarName, JarInfoUpdater updater) {
+ JarInfo cached = getJarInfo(jarTypeEnum, jarName);
+ if (cached == null) {
+ throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_JAR, jarName);
}
- return crud.save(jarInfo);
+ JarInfo copy = copyForWrite(cached);
+ updater.modify(copy);
+ return crud.save(copy);
}
public JarInfo removeJarInfo(JarTypeEnum jarTypeEnum, String jarName) {
@@ -111,4 +127,8 @@ public class JarInfoManager {
return listJarInfo().stream().filter(jarInfo -> jarTypeEnum == jarInfo.getJarType())
.collect(Collectors.toList());
}
+
+ public interface JarInfoUpdater {
+ void modify(JarInfo copyForWrite);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 005fe4f97a..62ec03ba9f 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -145,6 +145,10 @@ public class ComputedColumnDesc implements Serializable {
return ComputedColumnDesc.CC_PREFIX + columnName;
}
+ public String getIdentityCcName() {
+ return tableIdentity + "." + columnName;
+ }
+
private String handleLegacyCC(String expr, String rootFact, Set<String> aliasSet) {
try {
String ret = CalciteParser.insertAliasInExpr(expr, rootFact);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java
index 5c22e5754c..21acfc69e5 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java
@@ -27,11 +27,13 @@ import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.MissingRootPersistentEntity;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.Data;
import lombok.EqualsAndHashCode;
@@ -126,4 +128,10 @@ public class FusionModel extends RootPersistentEntity implements Serializable {
}
return null;
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + project + ResourceStore.FUSION_MODEL_RESOURCE_ROOT + "/" + resourceName()
+ + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java
index 45fdf65aae..6d94db46d2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java
@@ -89,14 +89,19 @@ public class FusionModelManager {
return model;
}
+ private FusionModel copyForWrite(FusionModel desc) {
+ return crud.copyForWrite(desc);
+ }
+
public FusionModel createModel(FusionModel desc) {
if (desc == null) {
throw new IllegalArgumentException();
}
- if (crud.contains(desc.resourceName()))
- throw new IllegalArgumentException("Fusion Model '" + desc.getAlias() + "' already exists");
+ FusionModel copy = copyForWrite(desc);
+ if (crud.contains(copy.resourceName()))
+ throw new IllegalArgumentException("Fusion Model '" + copy.getAlias() + "' already exists");
- return crud.save(desc);
+ return crud.save(copy);
}
public String getModelId(NativeQueryRealization realization) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 99c0d6c908..ec47608b85 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -22,6 +22,11 @@ import static org.apache.kylin.common.exception.ServerErrorCode.COLUMN_NOT_EXIST
import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_UPDATE_MODEL;
import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_JOIN_RELATIONSHIP_ERROR;
import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_NOT_EXIST;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_EXPRESSION;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_NAME;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.INDEX_PLAN;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.MODEL_DESC;
+import static org.apache.kylin.common.persistence.lock.ResourcePathParser.JOINER;
import java.io.Serializable;
import java.util.ArrayDeque;
@@ -484,6 +489,11 @@ public class NDataModel extends RootPersistentEntity {
return Collections.unmodifiableMap(aliasMap);
}
+ public void setAliasMap(Map<String, TableRef> aliasMap) {
+ this.aliasMap.clear();
+ this.aliasMap.putAll(aliasMap);
+ }
+
public Set<TableRef> getLookupTables() {
return lookupTableRefs;
}
@@ -1397,6 +1407,20 @@ public class NDataModel extends RootPersistentEntity {
return "/" + project + ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + name + MetadataConstants.FILE_SURFIX;
}
+ @Override
+ public List<String> getLockPaths(String ignored) {
+ List<String> lockPaths = new ArrayList<>();
+ lockPaths.add(getResourcePath());
+ lockPaths.add(JOINER.join("", project, INDEX_PLAN.value(), getUuid()));
+ lockPaths.add(JOINER.join("", project, MODEL_DESC.value(), alias));
+ for (ComputedColumnDesc ccDesc : computedColumnDescs) {
+ lockPaths.add(JOINER.join("", project, COMPUTE_COLUMN_NAME.value(), ccDesc.getIdentityCcName()));
+ lockPaths.add(JOINER.join("", project, COMPUTE_COLUMN_EXPRESSION.value(),
+ ccDesc.getInnerExpression().replace('/', '@')));
+ }
+ return lockPaths;
+ }
+
public Collection<NamedColumn> getAllSelectedColumns() {
Set<NamedColumn> selectedColumns = new HashSet<>();
for (NamedColumn namedColumn : allNamedColumns) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
index 8b9f00b0e1..0fe43411d8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
@@ -35,20 +35,21 @@ import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.exception.ModelBrokenException;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -117,6 +118,7 @@ public class NDataModelManager {
model.setJoinTables(entity.getJoinTables());
model.setDependencies(model.calcDependencies());
model.setModelType(entity.getModelType());
+ model.setLastModified(entity.getLastModified());
postModelBrokenEvent(entity);
}
@@ -241,6 +243,7 @@ public class NDataModelManager {
public NDataModel createDataModelDesc(NDataModel model, String owner) {
NDataModel copy = copyForWrite(model);
+ MemoryLockUtils.manuallyLockModule(project, ModuleLockEnum.MODEL, getStore());
checkDuplicateModel(model);
//check model count
@@ -283,21 +286,29 @@ public class NDataModelManager {
}
public NDataModel updateDataModel(String modelId, NDataModelUpdater updater) {
- val cached = getDataModelDesc(modelId);
+ NDataModel cached = getDataModelDesc(modelId);
if (cached == null) {
throw new KylinException(ErrorCodeServer.MODEL_ID_NOT_EXIST, modelId);
}
+ if (cached.isBroken()) {
+ cached = getDataModelDescWithoutInit(modelId);
+ }
val copy = copyForWrite(cached);
updater.modify(copy);
- return updateDataModelDesc(copy);
+ return saveModel(copy);
}
+ /**
+ * @deprecated Use updateDataModel(String modelId, NDataModelUpdater updater) instead.
+ */
+ @Deprecated
public NDataModel updateDataModelDesc(NDataModel desc) {
String name = desc.getUuid();
if (!crud.contains(desc.getUuid())) {
throw new IllegalArgumentException("Model '" + name + "' does not exist.");
}
- return saveModel(desc);
+ desc.init(config, project, getCCRelatedModels(desc));
+ return updateDataModel(desc.getUuid(), desc::copyPropertiesTo);
}
public NDataModel updateDataBrokenModelDesc(NDataModel desc) {
@@ -320,6 +331,10 @@ public class NDataModelManager {
return crud.copyForWrite(nDataModel);
}
+ public NDataModel copy(NDataModel nDataModel) {
+ return copyBySerialization(nDataModel);
+ }
+
/**
* copyBySerialization will not use cache
*/
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
index 64103f9c11..ebc1e893b9 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
@@ -31,22 +31,18 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import lombok.val;
-
/**
*/
public class NTableMetadataManager {
@@ -54,8 +50,6 @@ public class NTableMetadataManager {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(NTableMetadataManager.class);
- private static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<>(TableExtDesc.class);
-
public static NTableMetadataManager getInstance(KylinConfig config, String project) {
return config.getManager(project, NTableMetadataManager.class);
}
@@ -153,7 +147,14 @@ public class NTableMetadataManager {
return srcTableCrud.get(tableName);
}
+ public TableDesc copy(TableDesc tableDesc) {
+ return srcTableCrud.copyBySerialization(tableDesc);
+ }
+
public TableDesc copyForWrite(TableDesc tableDesc) {
+ if (tableDesc.getProject() == null) {
+ tableDesc.setProject(project);
+ }
return srcTableCrud.copyForWrite(tableDesc);
}
@@ -161,9 +162,16 @@ public class NTableMetadataManager {
return srcExtCrud.copyForWrite(tableExtDesc);
}
+ /**
+ * @deprecated Use updateTableDesc(String tableName, TableDescUpdater updater) instead.
+ */
+ @Deprecated
public void saveSourceTable(TableDesc srcTable) {
- srcTable.init(project);
- srcTableCrud.save(srcTable);
+ if (srcTableCrud.contains(srcTable.getIdentity())) {
+ updateTableDesc(srcTable.getIdentity(), srcTable::copyPropertiesTo);
+ } else {
+ createTableDesc(srcTable);
+ }
}
public void removeSourceTable(String tableIdentity) {
@@ -182,6 +190,35 @@ public class NTableMetadataManager {
srcTableCrud.reloadAll();
}
+ /**
+ * @deprecated Use updateTableDesc(String tableName, TableDescUpdater updater) instead.
+ */
+ @Deprecated
+ public void updateTableDesc(TableDesc tableDesc) {
+ updateTableDesc(tableDesc.getIdentity(), tableDesc::copyPropertiesTo);
+ }
+
+ public void updateTableDesc(String tableName, TableDescUpdater updater) {
+ TableDesc cached = getTableDesc(tableName);
+ if (cached == null) {
+ throw new IllegalStateException("tableDesc " + tableName + " does not exist");
+ }
+ TableDesc copy = copyForWrite(cached);
+ updater.modify(copy);
+ copy.init(project);
+ srcTableCrud.save(copy);
+ }
+
+ public void createTableDesc(TableDesc srcTable) {
+ srcTable.init(project);
+ TableDesc copy = copyForWrite(srcTable);
+ srcTableCrud.save(copy);
+ }
+
+ public interface TableDescUpdater {
+ void modify(TableDesc copyForWrite);
+ }
+
// ============================================================================
// TableExtDesc methods
// ============================================================================
@@ -239,38 +276,50 @@ public class NTableMetadataManager {
return srcTableCrud.getSerializer();
}
+ /**
+ * @deprecated Use updateTableExt(String tableName, TableExtDescUpdater updater) instead.
+ */
+ @Deprecated
public void saveTableExt(TableExtDesc tableExt) {
- if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
- throw new IllegalArgumentException();
+ if (srcExtCrud.contains(tableExt.getIdentity())) {
+ updateTableExt(tableExt.getIdentity(), tableExt::copyPropertiesTo);
+ } else {
+ createTableExt(tableExt);
}
+ }
- // what is this doing??
- String path = tableExt.getResourcePath();
- ResourceStore store = getStore();
- TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER);
- if (t != null && t.getIdentity() == null)
- store.deleteResource(path);
+ public void mergeAndUpdateTableExt(TableExtDesc origin, TableExtDesc other) {
+ updateTableExt(origin.getIdentity(), copyForWrite -> {
+ copyForWrite.setColumnStats(other.getAllColumnStats());
+ copyForWrite.setSampleRows(other.getSampleRows());
+ copyForWrite.setTotalRows(other.getTotalRows());
+ copyForWrite.setJodID(other.getJodID());
+ if (other.getOriginalSize() != -1) {
+ copyForWrite.setOriginalSize(other.getOriginalSize());
+ }
+ });
+ }
- srcExtCrud.save(tableExt);
+ public void updateTableExt(String tableName, TableExtDescUpdater updater) {
+ TableExtDesc cached = getOrCreateTableExt(tableName);
+ TableExtDesc copy = copyForWrite(cached);
+ updater.modify(copy);
+ srcExtCrud.save(copy);
}
- public void mergeAndUpdateTableExt(TableExtDesc origin, TableExtDesc other) {
- val copyForWrite = srcExtCrud.copyForWrite(origin);
- copyForWrite.setColumnStats(other.getAllColumnStats());
- copyForWrite.setSampleRows(other.getSampleRows());
- copyForWrite.setTotalRows(other.getTotalRows());
- copyForWrite.setJodID(other.getJodID());
- if (other.getOriginalSize() != -1) {
- copyForWrite.setOriginalSize(other.getOriginalSize());
+ public void createTableExt(TableExtDesc tableExt) {
+ if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
+ throw new IllegalArgumentException();
}
- saveTableExt(copyForWrite);
+ TableExtDesc copy = copyForWrite(tableExt);
+ srcExtCrud.save(copy);
}
public void saveOrUpdateTableExt(boolean isUpdate, TableExtDesc tableExt) {
if (isUpdate) {
mergeAndUpdateTableExt(tableExt, tableExt);
} else {
- saveTableExt(tableExt);
+ createTableExt(tableExt);
}
}
@@ -314,11 +363,7 @@ public class NTableMetadataManager {
return result;
}
- public void updateTableDesc(TableDesc tableDesc) {
- if (!srcTableCrud.contains(tableDesc.getIdentity())) {
- throw new IllegalStateException("tableDesc " + tableDesc.getName() + "does not exist");
- }
- saveSourceTable(tableDesc);
+ public interface TableExtDescUpdater {
+ void modify(TableExtDesc copyForWrite);
}
-
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 76dba6c87b..6ff06067d1 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -35,6 +35,9 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -47,9 +50,6 @@ import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import lombok.AllArgsConstructor;
import lombok.Getter;
@@ -374,6 +374,10 @@ public class TableDesc extends RootPersistentEntity implements Serializable, ISo
return project;
}
+ public void setProject(String project) {
+ this.project = project;
+ }
+
public String getName() {
return name == null ? null : name.toUpperCase(Locale.ROOT);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 931b86a5b1..aed99e2747 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -29,9 +29,13 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.base.Strings;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.MetadataConstants;
@@ -40,10 +44,7 @@ import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.base.Strings;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -171,7 +172,7 @@ public class TableExtDesc extends RootPersistentEntity implements Serializable {
}
public void setCardinality(String cardinality) {
- if (null == cardinality)
+ if (StringUtils.isEmpty(cardinality))
return;
String[] cardi = cardinality.split(",");
@@ -191,6 +192,16 @@ public class TableExtDesc extends RootPersistentEntity implements Serializable {
}
}
+ @Override
+ public <T extends RootPersistentEntity> void copyPropertiesTo(T copy) {
+ TableExtDesc extDesc = (TableExtDesc) copy;
+ extDesc.setColumnStats(this.getAllColumnStats()); // copyProperties cannot copy ColumnStats
+ for (Map.Entry<String, String> entry : this.getDataSourceProps().entrySet()) {
+ extDesc.addDataSourceProp(entry.getKey(), entry.getValue());
+ }
+ super.copyPropertiesTo(extDesc);
+ }
+
public enum RowCountStatus {
OK("ok"), TENTATIVE("tentative");
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java
index 821f242aea..5fb0f2f25a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java
@@ -151,7 +151,7 @@ public class ImportModelContext implements AutoCloseable {
List<TableDesc> tables = importTableMetadataManager.listAllTables();
for (TableDesc tableDesc : tables) {
- TableDesc newTable = targetTableMetadataManager.copyForWrite(tableDesc);
+ TableDesc newTable = targetTableMetadataManager.copy(tableDesc);
TableDesc originalTable = targetTableMetadataManager.getTableDesc(newTable.getIdentity());
newTable.setLastModified(System.currentTimeMillis());
if (Objects.isNull(originalTable)) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
index 5b941801fd..7eb5569f09 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
@@ -30,7 +30,11 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
@@ -38,9 +42,6 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.val;
public class NProjectManager {
@@ -108,10 +109,11 @@ public class NProjectManager {
ProjectInstance currentProject = getProject(projectName);
if (currentProject == null) {
+ MemoryLockUtils.manuallyLockModule("_global", ModuleLockEnum.PROJECT, getStore());
//circuit breaker
NCircuitBreaker.verifyProjectCreation(listAllProjects().size());
- currentProject = ProjectInstance.create(projectName, owner, description, overrideProps);
+ currentProject = copyForWrite(ProjectInstance.create(projectName, owner, description, overrideProps));
currentProject.initConfig(config);
} else {
throw new IllegalStateException("The project named " + projectName + "already exists");
@@ -166,7 +168,7 @@ public class NProjectManager {
});
}
- public ProjectInstance updateProject(ProjectInstance project) {
+ private ProjectInstance updateProject(ProjectInstance project) {
if (getProject(project.getName()) == null) {
throw new IllegalArgumentException("Project '" + project.getName() + "' does not exist!");
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java
index bd59b47bab..b87512f743 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java
@@ -91,7 +91,7 @@ public class ResourceGroupManager {
public void initResourceGroup() {
if (!resourceGroupInitialized()) {
- save(new ResourceGroup());
+ save(copyForWrite(new ResourceGroup()));
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
index 4cd63edd72..fed4def3c9 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
@@ -117,9 +117,14 @@ public class SourceUsageManager {
return crud.copyBySerialization(df);
}
+ public SourceUsageRecord copyForWrite(SourceUsageRecord df) {
+ return crud.copyForWrite(df);
+ }
+
public SourceUsageRecord createSourceUsageRecord(String resourceName, SourceUsageRecord record) {
record.setResPath(concatResourcePath(resourceName));
- return crud.save(record);
+ SourceUsageRecord copy = copyForWrite(record);
+ return crud.save(copy);
}
public SourceUsageRecord updateSourceUsageRecord(String resourceName, SourceUsageRecordUpdater updater) {
@@ -127,7 +132,7 @@ public class SourceUsageManager {
if (record == null) {
record = new SourceUsageRecord();
}
- SourceUsageRecord copy = copy(record);
+ SourceUsageRecord copy = copyForWrite(record);
updater.modify(copy);
return crud.save(copy);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java
index a2913262d8..3eaf5d8199 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java
@@ -33,9 +33,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import lombok.extern.slf4j.Slf4j;
@@ -96,18 +95,21 @@ public class DataParserManager {
if (parserInfo == null || StringUtils.isEmpty(parserInfo.getClassName())) {
throw new IllegalArgumentException("data parser info is null or class name is null");
}
- if (crud.contains(parserInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_PARSER, parserInfo.getClassName());
+ DataParserInfo copy = copyForWrite(parserInfo);
+ if (crud.contains(copy.resourceName())) {
+ throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_PARSER, copy.getClassName());
}
- parserInfo.updateRandomUuid();
- return crud.save(parserInfo);
+ copy.updateRandomUuid();
+ return crud.save(copy);
}
public DataParserInfo updateDataParserInfo(DataParserInfo parserInfo) {
- if (!crud.contains(parserInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_PARSER, parserInfo.getClassName());
+ DataParserInfo copy = copyForWrite(parserInfo);
+ if (!crud.contains(copy.resourceName())) {
+ throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_PARSER, copy.getClassName());
}
- return crud.save(parserInfo);
+ parserInfo.copyPropertiesTo(copy);
+ return crud.save(copy);
}
/**
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java
index 06b35a9c67..c2d3a2ee6e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java
@@ -75,23 +75,30 @@ public class KafkaConfigManager {
}
return crud.get(id);
}
+
+ private KafkaConfig copyForWrite(KafkaConfig kafkaConfig) {
+ return crud.copyForWrite(kafkaConfig);
+ }
public KafkaConfig createKafkaConfig(KafkaConfig kafkaConfig) {
if (kafkaConfig == null || StringUtils.isEmpty(kafkaConfig.getName())) {
throw new IllegalArgumentException();
}
- if (crud.contains(kafkaConfig.resourceName()))
+ KafkaConfig copy = copyForWrite(kafkaConfig);
+ if (crud.contains(copy.resourceName()))
throw new IllegalArgumentException("Kafka Config '" + kafkaConfig.getName() + "' already exists");
- kafkaConfig.updateRandomUuid();
- return crud.save(kafkaConfig);
+ copy.updateRandomUuid();
+ return crud.save(copy);
}
public KafkaConfig updateKafkaConfig(KafkaConfig kafkaConfig) {
- if (!crud.contains(kafkaConfig.resourceName())) {
+ KafkaConfig copy = copyForWrite(kafkaConfig);
+ if (!crud.contains(copy.resourceName())) {
throw new IllegalArgumentException("Kafka Config '" + kafkaConfig.getName() + "' does not exist.");
}
- return crud.save(kafkaConfig);
+ kafkaConfig.copyPropertiesTo(copy);
+ return crud.save(copy);
}
public KafkaConfig removeKafkaConfig(String tableIdentity) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
index 0bfe31724c..7ccd5ee3f3 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
@@ -83,26 +83,33 @@ public class StreamingManager {
crud.reloadAll();
return listAllStreaming();
}
+
+ private StreamingConfig copyForWrite(StreamingConfig streamingConfig) {
+ return crud.copyForWrite(streamingConfig);
+ }
public StreamingConfig createStreamingConfig(StreamingConfig streamingConfig) throws IOException {
if (streamingConfig == null || StringUtils.isEmpty(streamingConfig.getName())) {
throw new IllegalArgumentException();
}
- if (crud.contains(streamingConfig.resourceName()))
+ StreamingConfig copy = copyForWrite(streamingConfig);
+ if (crud.contains(copy.resourceName()))
throw new IllegalArgumentException("StreamingConfig '" + streamingConfig.getName() + "' already exists");
- streamingConfig.updateRandomUuid();
+ copy.updateRandomUuid();
- return crud.save(streamingConfig);
+ return crud.save(copy);
}
public StreamingConfig updateStreamingConfig(StreamingConfig desc) throws IOException {
if (desc.getUuid() == null || desc.getName() == null) {
throw new IllegalArgumentException("SteamingConfig Illegal.");
}
+ StreamingConfig copy = copyForWrite(desc);
if (!crud.contains(desc.resourceName())) {
throw new IllegalArgumentException("StreamingConfig '" + desc.getName() + "' does not exist.");
}
+ desc.copyPropertiesTo(copy);
return crud.save(desc);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java
index 0a00c78157..215b703144 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java
@@ -22,12 +22,11 @@ import static org.apache.kylin.common.persistence.ResourceStore.UPGRADE;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
public class GlobalAclVersionManager {
private static final Logger logger = LoggerFactory.getLogger(GlobalAclVersionManager.class);
@@ -64,10 +63,15 @@ public class GlobalAclVersionManager {
return crud.exists(GlobalAclVersion.VERSION_KEY_NAME);
}
+ private GlobalAclVersion copyForWrite(GlobalAclVersion globalAclVersion) {
+ return crud.copyForWrite(globalAclVersion);
+ }
+
public void save(GlobalAclVersion globalAclVersion) {
Preconditions.checkNotNull(globalAclVersion.getAclVersion());
+ GlobalAclVersion copy = copyForWrite(globalAclVersion);
if (!exists()) {
- crud.save(globalAclVersion);
+ crud.save(copy);
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java
index a56d4828be..fe459bf6a7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java
@@ -51,6 +51,8 @@ import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
+import static org.apache.kylin.common.persistence.ResourceStore.USER_ROOT;
+
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
@Getter
@@ -143,6 +145,11 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails, Us
this.setUuid(RandomUtil.randomUUIDStr());
}
+ @Override
+ public String getResourcePath() {
+ return USER_ROOT + "/" + username;
+ }
+
@Override
public String resourceName() {
return username;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
index bfb324667e..4f40d072df 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
@@ -31,13 +31,12 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
public class NKylinUserManager {
private static final Logger logger = LoggerFactory.getLogger(NKylinUserManager.class);
@@ -74,6 +73,10 @@ public class NKylinUserManager {
return crud.copyForWrite(user);
}
+ public ManagedUser copy(ManagedUser user) {
+ return crud.copyBySerialization(user);
+ }
+
public KylinConfig getConfig() {
return config;
}
@@ -107,16 +110,30 @@ public class NKylinUserManager {
return users;
}
- public void update(ManagedUser user) {
- ManagedUser exist = crud.get(user.getUsername());
+ public void createUser(ManagedUser user) {
ManagedUser copy = copyForWrite(user);
- if (exist != null) {
- copy.setLastModified(exist.getLastModified());
- copy.setMvcc(exist.getMvcc());
- }
crud.save(copy);
}
+ public void updateUser(String userName, UserUpdater updater) {
+ ManagedUser cached = get(userName);
+ ManagedUser copy = copyForWrite(cached);
+ updater.modify(copy);
+ crud.save(copy);
+ }
+
+ /**
+ * @deprecated Use updateUser(String userName, UserUpdater updater)
+ */
+ @Deprecated
+ public void update(ManagedUser user) {
+ if (exists(user.getUsername())) {
+ updateUser(user.getUsername(), user::copyPropertiesTo);
+ } else {
+ createUser(user);
+ }
+ }
+
public void delete(String username) {
crud.delete(username);
}
@@ -132,4 +149,8 @@ public class NKylinUserManager {
return user.getAuthorities().stream().map(SimpleGrantedAuthority::getAuthority).collect(Collectors.toSet());
}
+
+ public interface UserUpdater {
+ void modify(ManagedUser copy);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java
index 7a82c08e32..febccc3be6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java
@@ -137,7 +137,7 @@ public class NUserGroupManager {
String.format(Locale.ROOT, MsgPicker.getMsg().getUserGroupExist(), String.join(",", realGroups)));
}
for (String name : names) {
- UserGroup userGroup = new UserGroup(name);
+ UserGroup userGroup = copyForWrite(new UserGroup(name));
crud.save(userGroup);
}
}
@@ -148,7 +148,7 @@ public class NUserGroupManager {
throw new KylinException(DUPLICATE_USERGROUP_NAME,
String.format(Locale.ROOT, MsgPicker.getMsg().getUserGroupExist(), realGroupName));
}
- UserGroup userGroup = new UserGroup(name);
+ UserGroup userGroup = copyForWrite(new UserGroup(name));
crud.save(userGroup);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
index dc22af1698..aaa428b250 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
@@ -25,6 +25,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
+import static org.apache.kylin.common.persistence.ResourceStore.USER_GROUP_ROOT;
+
@SuppressWarnings("serial")
@Getter
@Setter
@@ -44,4 +46,9 @@ public class UserGroup extends RootPersistentEntity {
public String resourceName() {
return groupName;
}
+
+ @Override
+ public String getResourcePath() {
+ return USER_GROUP_ROOT + "/" + resourceName();
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java
index f5a29de063..aeda3c2f91 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.SpringContext;
@@ -50,8 +51,6 @@ import org.springframework.security.acls.model.PermissionGrantingStrategy;
import org.springframework.security.acls.model.Sid;
import org.springframework.security.core.context.SecurityContextHolder;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.val;
/**
@@ -165,7 +164,7 @@ public class AclManager {
parentAcl = readAclById(record.getParentDomainObjectInfo());
// Caution: Using a copied object before doing init, otherwise the original one will be polluted
- AclRecord recordCopy = crud.copyForWrite(record);
+ AclRecord recordCopy = crud.copyBySerialization(record);
recordCopy.init(parentAcl, aclPermissionFactory, permissionGrantingStrategy);
aclMaps.put(oid, new MutableAclRecord(recordCopy));
@@ -178,7 +177,7 @@ public class AclManager {
if (aclRecord != null) {
throw new AlreadyExistsException(String.format(Locale.ROOT, "ACL of %s exists!", objectIdentity));
}
- AclRecord record = newAclRecord(objectIdentity);
+ AclRecord record = copyForWrite(newAclRecord(objectIdentity));
crud.save(record);
logger.debug("ACL of {} created successfully.", objectIdentity);
@@ -199,11 +198,13 @@ public class AclManager {
logger.debug("ACL of {} deleted successfully.", objectIdentity);
}
+ /**
+ * @deprecated Use updateAcl(MutableAclRecord acl, AclRecordUpdater updater) instead.
+ */
+ @Deprecated
public MutableAcl updateAcl(MutableAcl mutableAcl) {
- AclRecord record = ((MutableAclRecord) mutableAcl).getAclRecord();
- crud.save(record);
- logger.debug("ACL of {} updated successfully.", mutableAcl.getObjectIdentity());
- return mutableAcl;
+ return updateAcl((MutableAclRecord) mutableAcl,
+ ((MutableAclRecord) mutableAcl).getAclRecord()::copyPropertiesTo);
}
// a NULL permission means to delete the ace
@@ -248,6 +249,7 @@ public class AclManager {
val copyForWrite = crud.copyForWrite(acl.getAclRecord());
updater.update(copyForWrite);
crud.save(copyForWrite);
+ logger.debug("ACL of {} updated successfully.", copyForWrite.getObjectIdentity());
return readAcl(acl.getObjectIdentity()); // here we are done
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java
index 21bab12643..1a95f4f57d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java
@@ -301,6 +301,6 @@ public class AclRecord extends RootPersistentEntity implements Acl, OwnershipAcl
@Override
public String getResourcePath() {
- return ResourceStore.ACL_ROOT + "/" + uuid;
+ return ResourceStore.ACL_ROOT + "/" + resourceName();
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java
index 08813373e1..608eb88fe5 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.security;
+import static org.apache.kylin.common.persistence.ResourceStore.ACL_GLOBAL_ROOT;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -121,6 +123,11 @@ public class UserAcl extends RootPersistentEntity {
return CollectionUtils.isNotEmpty(permissionMasks) && permissionMasks.contains(permissionMask);
}
+ @Override
+ public String getResourcePath() {
+ return ACL_GLOBAL_ROOT + "/" + username;
+ }
+
@Override
public String resourceName() {
return username;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
index 7355933597..d19f263141 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
@@ -111,7 +111,7 @@ public class UserAclManager {
public void addPermission(String name, Set<Permission> permissions) {
val username = filterUsername(name);
if (!username.isPresent()) {
- UserAcl userAcl = new UserAcl(name, permissions);
+ UserAcl userAcl = crud.copyForWrite(new UserAcl(name, permissions));
crud.save(userAcl);
} else {
UserAcl userAcl = get(username.get());
@@ -122,7 +122,7 @@ public class UserAclManager {
public void addDataQueryProject(String name, String project) {
val username = filterUsername(name);
if (!username.isPresent()) {
- UserAcl userAcl = new UserAcl(name, Collections.emptySet());
+ UserAcl userAcl = crud.copyForWrite(new UserAcl(name, Collections.emptySet()));
userAcl.addDataQueryProject(project);
crud.save(userAcl);
} else {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java b/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java
index 8dbcb9a2ce..20b3edead2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java
@@ -22,12 +22,11 @@ import java.lang.reflect.Method;
import java.util.Set;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.springframework.cglib.proxy.Enhancer;
import org.springframework.cglib.proxy.MethodInterceptor;
import org.springframework.cglib.proxy.MethodProxy;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.RequiredArgsConstructor;
import lombok.val;
@@ -39,7 +38,7 @@ public class BrokenEntityProxy implements MethodInterceptor {
"toString", "getMvcc", "setMvcc", "setConfig", "getConfig", "getModelAlias", "getModel",
"getRootFactTableName", "setRootFactTableName", "getJoinTables", "setJoinTables", "calcDependencies",
"getDependencies", "setDependencies", "setHandledAfterBroken", "isHandledAfterBroken", "getOwner",
- "setOwner", "isMultiPartitionModel",
+ "setOwner", "isMultiPartitionModel", "setLastModified",
// for broken fusion model
"getModelId", "getModelType", "isStreaming", "isFusionModel", "getModelTypeFromTable", "isAccessible",
"fusionModelStreamingPart", "fusionModelBatchPart", "setModelType", "getLastModified");
@@ -65,6 +64,11 @@ public class BrokenEntityProxy implements MethodInterceptor {
if (method.getName().equals("checkIsNotCachedAndShared")) {
return null;
}
+
+ if (method.getName().equals("getLockPaths")) {
+ return null;
+ }
+
throw new RuntimeException("call on Broken Entity's " + method.getName() + " method");
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
new file mode 100644
index 0000000000..f5ba147c92
--- /dev/null
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.cachesync;
+
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.user.ManagedUser;
+import org.apache.kylin.metadata.user.NKylinUserManager;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@MetadataInfo(onlyProps = true)
+class CachedCrudAssistTest {
+
+ @Test
+ void testCopyForWrite() {
+ NKylinUserManager manager = NKylinUserManager.getInstance(getTestConfig());
+
+ ManagedUser newUser = new ManagedUser("a", "a", false);
+
+ ManagedUser brokenUser = new ManagedUser("b", "b", false);
+ brokenUser.setBroken(true);
+ manager.createUser(brokenUser);
+
+ ManagedUser normalUser = new ManagedUser("c", "c", false);
+ manager.createUser(normalUser);
+
+ ManagedUser delUser = new ManagedUser("d", "d", false);
+ delUser.setMvcc(10);
+
+ // null entity
+ Assertions.assertNull(manager.copyForWrite(null));
+
+ // new entity
+ Assertions.assertEquals(newUser, manager.copyForWrite(newUser));
+
+ // broken entity
+ Assertions.assertEquals(brokenUser, manager.copyForWrite(brokenUser));
+
+ // normalEntity
+ Assertions.assertEquals(0, manager.copyForWrite(normalUser).getMvcc());
+
+ // insert a deleted user
+ Assertions.assertThrows(IllegalArgumentException.class, () -> manager.createUser(delUser));
+ }
+}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java
index 0151cac612..0fdefebc67 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java
@@ -34,9 +34,11 @@ import org.apache.commons.collections.ListUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.guava30.shaded.common.collect.BiMap;
+import org.apache.kylin.guava30.shaded.common.collect.Iterables;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexEntity;
@@ -44,18 +46,14 @@ import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.cube.model.RuleBasedIndex;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.collect.BiMap;
-import org.apache.kylin.guava30.shaded.common.collect.Iterables;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.val;
import lombok.var;
@@ -314,12 +312,11 @@ public class IndexPlanTest {
// test effect on index plan when project is updated
{
NProjectManager pm = NProjectManager.getInstance(getTestConfig());
- ProjectInstance p = pm.getProject(projectDefault);
- ProjectInstance newP = pm.copyForWrite(p);
- LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(newP.getOverrideKylinProps());
- overrideCopy.put("testkey", "testvalue1");
- newP.setOverrideKylinProps(overrideCopy);
- pm.updateProject(newP);
+ pm.updateProject(projectDefault, copyForWrite -> {
+ LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(copyForWrite.getOverrideKylinProps());
+ overrideCopy.put("testkey", "testvalue1");
+ copyForWrite.setOverrideKylinProps(overrideCopy);
+ });
Assert.assertEquals("testvalue1",
((KylinConfigExt) indexPlan.getConfig()).getExtendedOverrides().get("testkey"));
@@ -327,12 +324,11 @@ public class IndexPlanTest {
{
NProjectManager pm = NProjectManager.getInstance(getTestConfig());
- ProjectInstance p = pm.getProject(projectDefault);
- ProjectInstance newP = pm.copyForWrite(p);
- LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(newP.getOverrideKylinProps());
- overrideCopy.put("testkey", "testvalue2");
- newP.setOverrideKylinProps(overrideCopy);
- pm.updateProject(newP);
+ pm.updateProject(projectDefault, copyForWrite -> {
+ LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(copyForWrite.getOverrideKylinProps());
+ overrideCopy.put("testkey", "testvalue2");
+ copyForWrite.setOverrideKylinProps(overrideCopy);
+ });
Assert.assertEquals("testvalue2",
((KylinConfigExt) indexPlan.getConfig()).getExtendedOverrides().get("testkey"));
@@ -364,12 +360,11 @@ public class IndexPlanTest {
{
//test ProjectInstance trim k-v
NProjectManager pm = NProjectManager.getInstance(getTestConfig());
- ProjectInstance p = pm.getProject(projectDefault);
- ProjectInstance newP = pm.copyForWrite(p);
- LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(newP.getOverrideKylinProps());
- overrideCopy.put(" testkey ", " testvalue2 ");
- newP.setOverrideKylinProps(overrideCopy);
- pm.updateProject(newP);
+ pm.updateProject(projectDefault, copyForWrite -> {
+ LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(copyForWrite.getOverrideKylinProps());
+ overrideCopy.put(" testkey ", " testvalue2 ");
+ copyForWrite.setOverrideKylinProps(overrideCopy);
+ });
Assert.assertEquals("testvalue2",
((KylinConfigExt) indexPlan.getConfig()).getExtendedOverrides().get("testkey"));
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java
index dad5d97bfc..eec1754c98 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java
@@ -130,10 +130,9 @@ public class NDataLoadingRangeManagerTest extends NLocalFileMetadataTestCase {
String start = "2012-12-25 14:27:14.000";
String end = "2013-01-15 14:27:14.000";
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject("default");
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
- prjManager.updateProject(copy);
+ prjManager.updateProject("default", copyForWrite -> {
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
val loadingRange = createDataLoadingRange(DateFormat.stringToMillis(start), DateFormat.stringToMillis(end));
val ranges = dataLoadingRangeManager.getSegRangesToBuildForNewDataflow(loadingRange);
Assert.assertEquals(4, ranges.size());
@@ -165,13 +164,11 @@ public class NDataLoadingRangeManagerTest extends NLocalFileMetadataTestCase {
String end = "2013-01-15 14:27:14.000";
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject("default");
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().getVolatileRange().setVolatileRangeNumber(3L);
- copy.getSegmentConfig().getVolatileRange().setVolatileRangeEnabled(true);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
-
- prjManager.updateProject(copy);
+ prjManager.updateProject("default", copyForWrite -> {
+ copyForWrite.getSegmentConfig().getVolatileRange().setVolatileRangeNumber(3L);
+ copyForWrite.getSegmentConfig().getVolatileRange().setVolatileRangeEnabled(true);
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
val loadingRange = createDataLoadingRange(DateFormat.stringToMillis(start), DateFormat.stringToMillis(end));
val ranges = dataLoadingRangeManager.getSegRangesToBuildForNewDataflow(loadingRange);
@@ -213,11 +210,10 @@ public class NDataLoadingRangeManagerTest extends NLocalFileMetadataTestCase {
String end = "2012-01-04 20:33:39.000";
val loadingRange = createDataLoadingRange(DateFormat.stringToMillis(start), DateFormat.stringToMillis(end));
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject("default");
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().getAutoMergeTimeRanges().add(AutoMergeTimeEnum.YEAR);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
- prjManager.updateProject(copy);
+ prjManager.updateProject("default", copyForWrite -> {
+ copyForWrite.getSegmentConfig().getAutoMergeTimeRanges().add(AutoMergeTimeEnum.YEAR);
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
val ranges = dataLoadingRangeManager.getSegRangesToBuildForNewDataflow(loadingRange);
Assert.assertEquals(4, ranges.size());
//10/12/24 00:00 - 11/01/01
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java
index f1bf71ddb5..3196105f33 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java
@@ -56,7 +56,7 @@ public class NDataSegDetailsManagerTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(8, details.getLayouts().size());
Assert.assertSame(segment.getConfig().base(), details.getConfig().base());
- details = mgr.upsertForSegment(details);
+ details = mgr.updateDetails(segment, copyForWrite -> {});
details = mgr.getForSegment(segment);
Assert.assertNotNull(details);
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java
index 2cc42c9d91..a5e1b7d230 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java
@@ -26,11 +26,11 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.dbcp.BasicDataSourceFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.StringEntity;
-import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.junit.rule.TransactionExceptedException;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.junit.After;
@@ -42,7 +42,6 @@ import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.springframework.jdbc.core.JdbcTemplate;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -63,8 +62,7 @@ public class EnhancedUnitOfWorkTest extends NLocalFileMetadataTestCase {
+ "@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1,username=sa,password=");
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/UUID", new StringEntity(RandomUtil.randomUUIDStr()),
- StringEntity.serializer);
+ resourceStore.createMetaStoreUuidIfNotExist();
return null;
}, "");
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java
index d4e285ac6e..8b9a422f9e 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java
@@ -19,10 +19,12 @@
package org.apache.kylin.metadata.model;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.kylin.common.KapConfig;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
@@ -31,6 +33,7 @@ import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.NDataModel.Measure;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -275,4 +278,51 @@ public class NDataModelManagerTest extends NLocalFileMetadataTestCase {
NCircuitBreaker.stop();
}
}
+
+ @Test
+ public void createProjectParallel() {
+ KylinConfig conf = getTestConfig();
+ NDataModelManager manager = NDataModelManager.getInstance(conf, projectDefault);
+ int maxModelNum = manager.listAllModels().size() + 1;
+ getTestConfig().setProperty("kylin.circuit-breaker.threshold.model", String.valueOf(maxModelNum));
+
+ NCircuitBreaker.start(KapConfig.wrap(getTestConfig()));
+
+ NDataModel tmpModel = manager.listAllModels().stream().filter(model -> model.getComputedColumnDescs().isEmpty())
+ .findFirst().get();
+ List<NDataModel> newModels = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ newModels.add(cloneModel(tmpModel));
+ }
+ List<Thread> threads = new ArrayList<>();
+ newModels.forEach(model -> threads.add(new Thread(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ NDataModelManager mManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(),
+ projectDefault);
+ return mManager.createDataModelDesc(model, "ADMIN");
+ }, projectDefault);
+ })));
+ threads.forEach(Thread::start);
+ threads.forEach(thread -> {
+ try {
+ thread.join();
+ } catch (InterruptedException ignored) {
+ }
+ });
+
+ NCircuitBreaker.stop();
+ Assert.assertEquals(maxModelNum, manager.listAllModels().size());
+ }
+
+ private NDataModel cloneModel(NDataModel model) {
+ NDataModel newModel = new NDataModel(model);
+ String uuid = RandomUtil.randomUUIDStr();
+ newModel.setAlias(newModel.getAlias() + uuid);
+ newModel.setProject(model.getProject());
+ newModel.setUuid(uuid);
+ newModel.setMvcc(-1);
+ newModel.setComputedColumnDescs(new ArrayList<>());
+ return newModel;
+ }
+
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
index 4cfb808f39..750fcaf3a2 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata.project;
import java.util.Arrays;
+import java.util.Collections;
import java.util.LinkedHashMap;
import org.apache.hadoop.fs.FileSystem;
@@ -122,4 +123,35 @@ public class NProjectManagerTest extends NLocalFileMetadataTestCase {
project.getConfig().getExtendedOverrides().get("kylin.query.implicit-computed-column-convert"));
}
}
+
+ @Test
+ public void createProjectParallel() throws InterruptedException {
+ KylinConfig conf = getTestConfig();
+ NProjectManager manager = NProjectManager.getInstance(conf);
+ int projectNum = manager.listAllProjects().size();
+ getTestConfig().setProperty("kylin.circuit-breaker.threshold.project", String.valueOf(projectNum + 1));
+
+ NCircuitBreaker.start(KapConfig.wrap(getTestConfig()));
+
+ Thread t1 = new Thread(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ NProjectManager pManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return pManager.createProject("project_tmp_1", "ADMIN", "", new LinkedHashMap<>());
+ }, "_global");
+ });
+ Thread t2 = new Thread(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ NProjectManager pManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return pManager.createProject("project_tmp_2", "ADMIN", "", new LinkedHashMap<>());
+ }, "_global");
+ });
+
+ t1.start();
+ t2.start();
+ t1.join();
+ t2.join();
+
+ NCircuitBreaker.stop();
+ Assert.assertEquals(projectNum + 1, manager.listAllProjects().size());
+ }
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java b/src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java
new file mode 100644
index 0000000000..7116330e1a
--- /dev/null
+++ b/src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.model;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.common.persistence.lock.ResourcePathParser;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+class NDataModelTest {
+
+ @Test
+ void testModelLockPath() {
+ NDataModel model = new NDataModel();
+ ComputedColumnDesc cc = new ComputedColumnDesc();
+ cc.setColumnName("cc1");
+ cc.setTableIdentity("ssb.lineOrder");
+ cc.setInnerExpression("a / 1 + a / 2 + a / 4");
+ ReflectionTestUtils.setField(model, "project", "default");
+ ReflectionTestUtils.setField(model, "computedColumnDescs", Collections.singletonList(cc));
+ List<ResourcePathParser.ResourcePath> resourcePaths = new ArrayList<>();
+ for (String lockPath : model.getLockPaths()) {
+ resourcePaths.add(ResourcePathParser.parseResourcePath(lockPath));
+ }
+ Assertions.assertEquals(5, resourcePaths.size());
+ Assertions.assertEquals(cc.getTableIdentity() + "." + cc.getColumnName(), resourcePaths.get(3).getResource1());
+
+ String escapedExpression = "a @ 1 + a @ 2 + a @ 4";
+ Assertions.assertEquals(escapedExpression, resourcePaths.get(4).getResource1());
+ }
+}
diff --git a/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java b/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
index 808af78f71..cacd16a852 100644
--- a/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
+++ b/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
@@ -20,25 +20,21 @@ package org.apache.kylin.sdk.datasource;
import java.util.List;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.sdk.datasource.framework.JdbcConnectorTest;
import org.junit.Assert;
import org.junit.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
public class PushdownRunnerSDKImplTest extends JdbcConnectorTest {
@Test
public void testExecuteQuery() throws Exception {
NProjectManager npr = NProjectManager.getInstance(getTestConfig());
- ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase("SSB");
- npr.updateProject(projectInstance);
+ npr.updateProject("default", copyForWrite -> copyForWrite.setDefaultDatabase("SSB"));
PushDownRunnerSDKImpl pushDownRunnerSDK = new PushDownRunnerSDKImpl();
- pushDownRunnerSDK.init(getTestConfig(), projectInstance.getName());
+ pushDownRunnerSDK.init(getTestConfig(), "default");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
String sql = "select count(*) from LINEORDER";
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
index 4f7f16afb0..1f40e7fc83 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
@@ -370,7 +370,7 @@ public class TableExtService extends BasicService {
}
});
return tableResponse;
- }, project, 1);
+ }, project, 2);
}
private List<Pair<TableDesc, TableExtDesc>> extractTableMeta(String[] tables, String project,
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index b2ae72706b..90cdfa7057 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -61,8 +61,8 @@ import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -74,6 +74,7 @@ import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.AddS3CredentialToSparkBroadcastEventNotifier;
import org.apache.kylin.common.persistence.transaction.TransactionException;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -84,6 +85,19 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.ShellException;
import org.apache.kylin.constants.AclConstants;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.base.Strings;
+import org.apache.kylin.guava30.shaded.common.collect.HashMultimap;
+import org.apache.kylin.guava30.shaded.common.collect.LinkedHashMultimap;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.MapDifference;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Multimap;
+import org.apache.kylin.guava30.shaded.common.collect.SetMultimap;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.graph.Graph;
+import org.apache.kylin.guava30.shaded.common.graph.Graphs;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -170,20 +184,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.base.Strings;
-import org.apache.kylin.guava30.shaded.common.collect.HashMultimap;
-import org.apache.kylin.guava30.shaded.common.collect.LinkedHashMultimap;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.MapDifference;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Multimap;
-import org.apache.kylin.guava30.shaded.common.collect.SetMultimap;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.graph.Graph;
-import org.apache.kylin.guava30.shaded.common.graph.Graphs;
-
import lombok.val;
import lombok.var;
@@ -302,26 +302,19 @@ public class TableService extends BasicService {
if (origTable == null || origTable.getProject() == null) {
nTableDesc.setUuid(RandomUtil.randomUUIDStr());
nTableDesc.setLastModified(0);
+ tableMetaMgr.saveSourceTable(nTableDesc);
} else {
- nTableDesc.setUuid(origTable.getUuid());
- nTableDesc.setLastModified(origTable.getLastModified());
- nTableDesc.setIncrementLoading(origTable.isIncrementLoading());
- nTableDesc.setTableComment(tableDesc.getTableComment());
+ tableMetaMgr.updateTableDesc(nTableDesc.getIdentity(), copyForWrite -> {
+ nTableDesc.copyPropertiesTo(copyForWrite);
+
+ copyForWrite.setUuid(origTable.getUuid());
+ copyForWrite.setLastModified(origTable.getLastModified());
+ copyForWrite.setIncrementLoading(origTable.isIncrementLoading());
+ copyForWrite.setTableComment(tableDesc.getTableComment());
+ });
}
- tableMetaMgr.saveSourceTable(nTableDesc);
if (extDesc != null) {
- TableExtDesc origExt = tableMetaMgr.getTableExtIfExists(tableDesc);
- TableExtDesc nTableExtDesc = new TableExtDesc(extDesc);
- if (origExt == null || origExt.getProject() == null) {
- nTableExtDesc.setUuid(RandomUtil.randomUUIDStr());
- nTableExtDesc.setLastModified(0);
- } else {
- nTableExtDesc.setUuid(origExt.getUuid());
- nTableExtDesc.setLastModified(origExt.getLastModified());
- nTableExtDesc.setMvcc(origExt.getMvcc());
- nTableExtDesc.setOriginalSize(origExt.getOriginalSize());
- }
val colNameMap = Stream.of(nTableDesc.getColumns())
.collect(Collectors.toMap(ColumnDesc::getName, col -> {
try {
@@ -330,11 +323,28 @@ public class TableService extends BasicService {
return Integer.MAX_VALUE;
}
}));
- nTableExtDesc.getAllColumnStats()
- .sort(Comparator.comparing(stat -> colNameMap.getOrDefault(stat.getColumnName(), -1)));
- nTableExtDesc.init(project);
-
- tableMetaMgr.saveTableExt(nTableExtDesc);
+ TableExtDesc origExt = tableMetaMgr.getTableExtIfExists(tableDesc);
+ TableExtDesc nTableExtDesc = new TableExtDesc(extDesc);
+ if (origExt == null || origExt.getProject() == null) {
+ nTableExtDesc.setUuid(RandomUtil.randomUUIDStr());
+ nTableExtDesc.setLastModified(0);
+ nTableExtDesc.getAllColumnStats()
+ .sort(Comparator.comparing(stat -> colNameMap.getOrDefault(stat.getColumnName(), -1)));
+ nTableExtDesc.init(project);
+ tableMetaMgr.saveTableExt(nTableExtDesc);
+ } else {
+ tableMetaMgr.updateTableExt(nTableExtDesc.getIdentity(), copyForWrite -> {
+ nTableExtDesc.copyPropertiesTo(copyForWrite);
+
+ copyForWrite.setUuid(origExt.getUuid());
+ copyForWrite.setLastModified(origExt.getLastModified());
+ copyForWrite.setMvcc(origExt.getMvcc());
+ copyForWrite.setOriginalSize(origExt.getOriginalSize());
+ copyForWrite.getAllColumnStats()
+ .sort(Comparator.comparing(stat -> colNameMap.getOrDefault(stat.getColumnName(), -1)));
+ copyForWrite.init(project);
+ });
+ }
if (!broadcastS3Conf.contains(extDesc.getS3RoleCredentialInfo())) {
addAndBroadcastSparkSession(extDesc.getS3RoleCredentialInfo());
broadcastS3Conf.add(extDesc.getS3RoleCredentialInfo());
@@ -854,8 +864,8 @@ public class TableService extends BasicService {
.collect(Collectors.toSet());
if (tableDesc.getDatabase().equals(projectInstance.getDefaultDatabase())
&& !databases.contains(projectInstance.getDefaultDatabase())) {
- projectInstance.setDefaultDatabase(ProjectInstance.DEFAULT_DATABASE);
- npr.updateProject(projectInstance);
+ npr.updateProject(project,
+ copyForWrite -> copyForWrite.setDefaultDatabase(ProjectInstance.DEFAULT_DATABASE));
}
return tableDesc.getIdentity();
@@ -1229,19 +1239,21 @@ public class TableService extends BasicService {
public Pair<String, List<String>> reloadTable(String projectName, String tableIdentity, boolean needSample,
int maxRows, boolean needBuild, int priority, String yarnQueue) {
aclEvaluate.checkProjectWritePermission(projectName);
- return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- Pair<String, List<String>> pair = new Pair<>();
- List<String> buildingJobs = innerReloadTable(projectName, tableIdentity, needBuild, null);
- pair.setSecond(buildingJobs);
- if (needSample && maxRows > 0) {
- List<String> jobIds = tableSamplingService.sampling(Sets.newHashSet(tableIdentity), projectName,
- maxRows, priority, yarnQueue, null);
- if (CollectionUtils.isNotEmpty(jobIds)) {
- pair.setFirst(jobIds.get(0));
- }
- }
- return pair;
- }, projectName);
+ UnitOfWorkParams<Pair<String, List<String>>> params = UnitOfWorkParams.<Pair<String, List<String>>> builder()
+ .unitName(projectName).processor(() -> {
+ Pair<String, List<String>> pair = new Pair<>();
+ List<String> buildingJobs = innerReloadTable(projectName, tableIdentity, needBuild, null);
+ pair.setSecond(buildingJobs);
+ if (needSample && maxRows > 0) {
+ List<String> jobIds = tableSamplingService.sampling(Sets.newHashSet(tableIdentity), projectName,
+ maxRows, priority, yarnQueue, null);
+ if (CollectionUtils.isNotEmpty(jobIds)) {
+ pair.setFirst(jobIds.get(0));
+ }
+ }
+ return pair;
+ }).build();
+ return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(params);
}
public Pair<String, List<String>> reloadAWSTableCompatibleCrossAccount(String projectName,
diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
index 2d65b47f59..4e6e9c53a6 100644
--- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
+++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
@@ -114,10 +114,6 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase {
ReflectionTestUtils.setField(accessService, "userAclService", userAclService);
ReflectionTestUtils.setField(accessService, "userService", userService);
- val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject(PROJECT);
- val copy = prjManager.copyForWrite(prj);
- prjManager.updateProject(copy);
Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin"));
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString()))
.thenReturn(false);
diff --git a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json
index 65cf8658a2..0dab9315f1 100644
--- a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json
+++ b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json
@@ -6,7 +6,6 @@
"model_type" : "STREAMING",
"alias" : "stream_merge",
"owner" : "ADMIN",
- "model_type": "STREAMING",
"config_last_modifier" : null,
"config_last_modified" : 0,
"description" : "",
diff --git a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json
index 036acaf634..9707ed941d 100644
--- a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json
+++ b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json
@@ -6,7 +6,6 @@
"model_type" : "STREAMING",
"alias" : "stream_merge1",
"owner" : "ADMIN",
- "model_type": "STREAMING",
"config_last_modifier" : null,
"config_last_modified" : 0,
"description" : "",
diff --git a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json
index 877de9a3c5..d87d6e0f0c 100644
--- a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json
+++ b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json
@@ -6,7 +6,6 @@
"model_type" : "STREAMING",
"alias" : "stream_merge2",
"owner" : "ADMIN",
- "model_type": "STREAMING",
"config_last_modifier" : null,
"config_last_modified" : 0,
"description" : "",
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java
index 5ec27dae8f..8f1256860a 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java
@@ -96,10 +96,9 @@ public class AutoMergeTest extends NLocalFileMetadataTestCase {
val dataflowManager = NDataflowManager.getInstance(getTestConfig(), DEFAULT_PROJECT);
val df = dataflowManager.getDataflowByModelAlias("nmodel_basic");
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject(DEFAULT_PROJECT);
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
- prjManager.updateProject(copy);
+ prjManager.updateProject(DEFAULT_PROJECT, copyForWrite -> {
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
SegmentAutoMergeUtil.autoMergeSegments(DEFAULT_PROJECT, df.getUuid(), "ADMIN");
}
diff --git a/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java b/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java
index 1ffa8078ea..09a16ebf76 100644
--- a/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java
+++ b/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java
@@ -35,12 +35,14 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ImageDesc;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.StringEntity;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.JdbcAuditLogStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.tool.MetadataTool;
import org.junit.After;
import org.junit.Assert;
@@ -49,7 +51,6 @@ import org.junit.Ignore;
import org.junit.Test;
import org.springframework.jdbc.core.JdbcTemplate;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -69,6 +70,7 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
+ "@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1,username=sa,password=");
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/UUID", null, false);
resourceStore.checkAndPutResource("/UUID", new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
return null;
@@ -94,6 +96,10 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
queryResourceStore.catchup();
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4", null, false);
resourceStore.checkAndPutResource("/p0/path1", ByteSource.wrap("path1".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path2", ByteSource.wrap("path2".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path3", ByteSource.wrap("path3".getBytes(charset)), -1);
@@ -107,6 +113,10 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
public void testMetadataCatchupWithBackup() throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4", null, false);
resourceStore.checkAndPutResource("/p0/path1", ByteSource.wrap("path1".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path2", ByteSource.wrap("path2".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path3", ByteSource.wrap("path3".getBytes(charset)), -1);
@@ -122,6 +132,13 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path5", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path6", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path7", null, false);
resourceStore.checkAndPutResource("/p0/path1", ByteSource.wrap("path1".getBytes(charset)), 0);
resourceStore.checkAndPutResource("/p0/path2", ByteSource.wrap("path2".getBytes(charset)), 0);
resourceStore.checkAndPutResource("/p0/path3", ByteSource.wrap("path3".getBytes(charset)), 0);
@@ -144,6 +161,10 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
public void testMetadata_RemoveAuditLog_Restore() throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1.json", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2.json", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3.json", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4.json", null, false);
resourceStore.checkAndPutResource("/_global/project/p0.json", ByteSource
.wrap("{ \"uuid\": \"1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b\"}".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path1.json",
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 92ddf793a6..29a8076cbf 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -89,9 +89,11 @@ import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
import lombok.val;
+import lombok.extern.slf4j.Slf4j;
@Controller
@RequestMapping(value = "/api/tables", produces = { HTTP_VND_APACHE_KYLIN_JSON })
+@Slf4j
public class NTableController extends NBasicController {
private static final String TABLE = "table";
@@ -231,7 +233,8 @@ public class NTableController extends NBasicController {
LoadTableResponse loadTableResponse = tableExtService.loadTablesWithShortCircuit(tableLoadRequest);
- if (!loadTableResponse.getNeedRealSampling().isEmpty() && Boolean.TRUE.equals(tableLoadRequest.getNeedSampling())) {
+ if (!loadTableResponse.getNeedRealSampling().isEmpty()
+ && Boolean.TRUE.equals(tableLoadRequest.getNeedSampling())) {
TableSamplingService.checkSamplingRows(tableLoadRequest.getSamplingRows());
tableSamplingService.sampling(loadTableResponse.getNeedRealSampling(), tableLoadRequest.getProject(),
tableLoadRequest.getSamplingRows(), tableLoadRequest.getPriority(), tableLoadRequest.getYarnQueue(),
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index e316a9599b..ae7f237265 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -132,6 +132,10 @@ public class ModelRequest extends NDataModel implements ModelInsensitiveRequest
private transient BiFunction<TableDesc, Boolean, Collection<ColumnDesc>> columnsFetcher = TableRef::filterColumns;
+ public BiFunction<TableDesc, Boolean, Collection<ColumnDesc>> getColumnsFetcher() {
+ return columnsFetcher != null ? columnsFetcher : TableRef::filterColumns;
+ }
+
public ModelRequest() {
super();
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index ba699d6c8d..5e2143a39a 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -84,7 +84,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -124,6 +123,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkContext;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
@@ -161,7 +161,6 @@ import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataLayout;
import org.apache.kylin.metadata.cube.model.NDataLoadingRange;
import org.apache.kylin.metadata.cube.model.NDataLoadingRangeManager;
-import org.apache.kylin.metadata.cube.model.NDataSegDetails;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -2024,16 +2023,19 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public NDataModel createModel(String project, ModelRequest modelRequest) {
checkNewModels(project, Lists.newArrayList(modelRequest));
- return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- NDataModel model = saveModel(project, modelRequest);
- modelRequest.setUuid(model.getUuid());
- updateExcludedCheckerResult(project, modelRequest);
- // enable second storage
- if (modelRequest.isWithSecondStorage() && !SecondStorageUtil.isModelEnable(project, model.getId())) {
- SecondStorageUtil.initModelMetaData(project, model.getId());
- }
- return getManager(NDataModelManager.class, project).getDataModelDesc(model.getUuid());
- }, project);
+ UnitOfWorkParams<NDataModel> params = UnitOfWorkParams.<NDataModel> builder().unitName(project)
+ .processor(() -> {
+ NDataModel model = saveModel(project, modelRequest);
+ modelRequest.setUuid(model.getUuid());
+ updateExcludedCheckerResult(project, modelRequest);
+ // enable second storage
+ if (modelRequest.isWithSecondStorage()
+ && !SecondStorageUtil.isModelEnable(project, model.getId())) {
+ SecondStorageUtil.initModelMetaData(project, model.getId());
+ }
+ return getManager(NDataModelManager.class, project).getDataModelDesc(model.getUuid());
+ }).build();
+ return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(params);
}
private NDataModel doCheckBeforeModelSave(String project, ModelRequest modelRequest) {
@@ -2442,10 +2444,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
NDataflow dataflow = dfManger.getDataflow(modelId);
for (String segmentId : segmentIds) {
NDataSegment seg = dataflow.getSegment(segmentId);
- NDataSegDetails segDetails = seg.getSegDetails();
- List<NDataLayout> layouts = new LinkedList<>(segDetails.getAllLayouts());
- layouts.removeIf(layout -> indexIds.contains(layout.getLayoutId()));
- dfManger.updateDataflowDetailsLayouts(seg, layouts);
+ dfManger.updateDataflowDetailsLayouts(seg, indexIds, Collections.emptyList());
}
getManager(NIndexPlanManager.class, project).updateIndexPlan(dataflow.getUuid(),
IndexPlan::removeTobeDeleteIndexIfNecessary);
@@ -3049,6 +3048,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
String[] segmentIds) {
aclEvaluate.checkProjectOperationPermission(project);
SecondStorageJobUtil.validateSegment(project, model, Arrays.asList(segmentIds));
+ UnitOfWork.get().doBeforeUpdate(
+ () -> SecondStorageJobUtil.validateSegment(project, model, Arrays.asList(segmentIds)));
checkSegmentsExistById(model, project, segmentIds);
checkSegmentsStatus(model, project, segmentIds, SegmentStatusEnumToDisplay.LOADING,
SegmentStatusEnumToDisplay.REFRESHING, SegmentStatusEnumToDisplay.MERGING,
@@ -3071,67 +3072,69 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public BuildBaseIndexResponse updateDataModelSemantic(String project, ModelRequest request, boolean isCheckFlat) {
try {
- return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- aclEvaluate.checkProjectWritePermission(project);
- semanticUpdater.expandModelRequest(request);
- checkModelRequest(request);
- checkModelPermission(project, request.getUuid());
- validatePartitionDateColumn(request);
-
- val modelId = request.getUuid();
- val modelManager = getManager(NDataModelManager.class, project);
- val originModel = modelManager.getDataModelDesc(modelId);
-
- val copyModel = modelManager.copyForWrite(originModel);
- UpdateImpact updateImpact = semanticUpdater.updateModelColumns(copyModel, request, true);
- copyModel.init(modelManager.getConfig(), project, modelManager.getCCRelatedModels(copyModel));
-
- BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(originModel,
- needHandleBaseIndexType(request));
-
- preProcessBeforeModelSave(copyModel, project);
- val updated = modelManager.updateDataModelDesc(copyModel);
-
- // 1. delete old internal measure
- // 2. expand new measures
- // the ordering of the two steps is important as tomb internal measure
- // should not be used to construct new expandable measures
- semanticUpdater.deleteExpandableMeasureInternalMeasures(updated);
- semanticUpdater.expandExpandableMeasure(updated);
- preProcessBeforeModelSave(updated, project);
- getManager(NDataModelManager.class, project).updateDataModelDesc(updated);
-
- indexPlanService.updateForMeasureChange(project, modelId, updateImpact.getInvalidMeasures(),
- updateImpact.getReplacedMeasures());
- Set<Integer> affectedSet = updateImpact.getAffectedIds();
- val affectedLayoutSet = getAffectedLayouts(project, modelId, affectedSet);
- if (affectedLayoutSet.size() > 0)
- indexPlanService.reloadLayouts(project, modelId, affectedLayoutSet);
- indexPlanService.clearShardColIfNotDim(project, modelId);
-
- var newModel = modelManager.getDataModelDesc(modelId);
-
- checkIndexColumnExist(project, modelId, originModel);
-
- if (isCheckFlat) {
- checkFlatTableSql(newModel);
- }
-
- val needBuild = semanticUpdater.doHandleSemanticUpdate(project, modelId, originModel,
- request.getStart(), request.getEnd());
-
- updateExcludedCheckerResult(project, request);
- baseIndexUpdater.setSecondStorageEnabled(request.isWithSecondStorage());
- BuildBaseIndexResponse baseIndexResponse = baseIndexUpdater.update(indexPlanService);
- if (!request.isSaveOnly() && (needBuild || baseIndexResponse.hasIndexChange())) {
- semanticUpdater.buildForModel(project, modelId);
- }
- modelChangeSupporters.forEach(listener -> listener.onUpdate(project, modelId));
-
- changeSecondStorageIfNeeded(project, request, () -> !semanticUpdater.isSignificantChange(originModel,
- modelManager.getDataModelDesc(modelId)));
- return baseIndexResponse;
- }, project);
+ UnitOfWorkParams<BuildBaseIndexResponse> params = UnitOfWorkParams.<BuildBaseIndexResponse> builder()
+ .unitName(project).processor(() -> {
+ aclEvaluate.checkProjectWritePermission(project);
+ semanticUpdater.expandModelRequest(request);
+ checkModelRequest(request);
+ checkModelPermission(project, request.getUuid());
+ validatePartitionDateColumn(request);
+
+ val modelId = request.getUuid();
+ val modelManager = getManager(NDataModelManager.class, project);
+ val originModel = modelManager.getDataModelDesc(modelId);
+
+ val copyModel = modelManager.copyForWrite(originModel);
+ UpdateImpact updateImpact = semanticUpdater.updateModelColumns(copyModel, request, true);
+ copyModel.init(modelManager.getConfig(), project, modelManager.getCCRelatedModels(copyModel));
+
+ BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(originModel,
+ needHandleBaseIndexType(request));
+
+ preProcessBeforeModelSave(copyModel, project);
+ val updated = modelManager.updateDataModelDesc(copyModel);
+
+ // 1. delete old internal measure
+ // 2. expand new measures
+ // the ordering of the two steps is important as tomb internal measure
+ // should not be used to construct new expandable measures
+ semanticUpdater.deleteExpandableMeasureInternalMeasures(updated);
+ semanticUpdater.expandExpandableMeasure(updated);
+ preProcessBeforeModelSave(updated, project);
+ getManager(NDataModelManager.class, project).updateDataModelDesc(updated);
+
+ indexPlanService.updateForMeasureChange(project, modelId, updateImpact.getInvalidMeasures(),
+ updateImpact.getReplacedMeasures());
+ Set<Integer> affectedSet = updateImpact.getAffectedIds();
+ val affectedLayoutSet = getAffectedLayouts(project, modelId, affectedSet);
+ if (affectedLayoutSet.size() > 0)
+ indexPlanService.reloadLayouts(project, modelId, affectedLayoutSet);
+ indexPlanService.clearShardColIfNotDim(project, modelId);
+
+ var newModel = modelManager.getDataModelDesc(modelId);
+
+ checkIndexColumnExist(project, modelId, originModel);
+
+ if (isCheckFlat) {
+ checkFlatTableSql(newModel);
+ }
+
+ val needBuild = semanticUpdater.doHandleSemanticUpdate(project, modelId, originModel,
+ request.getStart(), request.getEnd());
+
+ updateExcludedCheckerResult(project, request);
+ baseIndexUpdater.setSecondStorageEnabled(request.isWithSecondStorage());
+ BuildBaseIndexResponse baseIndexResponse = baseIndexUpdater.update(indexPlanService);
+ if (!request.isSaveOnly() && (needBuild || baseIndexResponse.hasIndexChange())) {
+ semanticUpdater.buildForModel(project, modelId);
+ }
+ modelChangeSupporters.forEach(listener -> listener.onUpdate(project, modelId));
+
+ changeSecondStorageIfNeeded(project, request, () -> !semanticUpdater
+ .isSignificantChange(originModel, modelManager.getDataModelDesc(modelId)));
+ return baseIndexResponse;
+ }).build();
+ return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(params);
} catch (TransactionException te) {
Throwable root = te.getCause();
if (root instanceof RuntimeException) {
@@ -3363,7 +3366,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
NDataModelManager modelMgr = getManager(NDataModelManager.class, project);
semanticUpdater.updateModelColumns(broken, modelRequest);
- NDataModel model = modelMgr.updateDataModelDesc(broken);
+ NDataModel copy = modelManager.copyForWrite(broken);
+ NDataModel model = modelMgr.updateDataModelDesc(copy);
modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
modelChangeSupporters.forEach(listener -> listener.onUpdateSingle(project, model.getUuid()));
saveDateFormatIfNotExist(project, model.getUuid(), format);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java
index 73b9a94fb7..cd4fae53ef 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java
@@ -345,12 +345,13 @@ public class IndexPlanServiceTest extends SourceTestCase {
val readySegs = dfManager.getDataflow(id).getSegments(SegmentStatusEnum.READY, SegmentStatusEnum.WARNING);
val last = readySegs.getLatestReadySegment().getSegDetails();
List<NDataLayout> lists = Lists.newArrayList(last.getLayouts());
+ List<Long> toAddLayout = new ArrayList<>();
saved.getRuleBasedIndex().getLayoutIdMapping().forEach(layoutId -> {
if (saved.getLayoutEntity(layoutId).getDimsIds().size() > 1) {
- lists.add(NDataLayout.newDataLayout(last, layoutId));
+ toAddLayout.add(layoutId);
}
});
- dfManager.updateDataflowDetailsLayouts(readySegs.getLatestReadySegment(), lists);
+ dfManager.updateDataflowDetailsLayouts(readySegs.getLatestReadySegment(), Collections.emptyList(), toAddLayout);
indexPlanManager.updateIndexPlan(id, copyForWrite -> {
copyForWrite.markIndexesToBeDeleted(copyForWrite.getId(), saved.getRuleBaseLayouts().stream()
.filter(layoutEntity -> layoutEntity.getColOrder().size() == 3).collect(Collectors.toSet()));
@@ -971,7 +972,9 @@ public class IndexPlanServiceTest extends SourceTestCase {
});
Assert.assertEquals(WARNING, dataflowManager.getDataflow(modelId).getLastSegment().getStatus());
- dataflowManager.updateDataflowDetailsLayouts(df.getLastSegment(), Lists.newArrayList());
+ List<Long> toRemoveLayouts = df.getLastSegment().getSegDetails().getAllLayouts().stream()
+ .map(NDataLayout::getLayoutId).collect(Collectors.toList());
+ dataflowManager.updateDataflowDetailsLayouts(df.getLastSegment(), toRemoveLayouts, Collections.emptyList());
Assert.assertEquals(READY, dataflowManager.getDataflow(modelId).getLastSegment().getStatus());
}
@@ -1270,8 +1273,8 @@ public class IndexPlanServiceTest extends SourceTestCase {
//remove tobedelete layout from seg1
val newDf = dfManager.getDataflow(modelId);
- dfManager.updateDataflowDetailsLayouts(newDf.getSegment(seg1.getId()), layouts1.stream()
- .filter(layout -> layout.getLayoutId() != tobeDeleteLayoutId).collect(Collectors.toList()));
+ dfManager.updateDataflowDetailsLayouts(newDf.getSegment(seg1.getId()),
+ Collections.singletonList(tobeDeleteLayoutId), Collections.emptyList());
//segment1
Assert.assertEquals(1,
indexPlanService.getIndexGraph(getProject(), modelId, 100).getSegmentToComplementCount());
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 58beefba66..8640e901d9 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -1761,8 +1761,7 @@ public class ModelServiceTest extends SourceTestCase {
//remove tobedelete layout from seg1
val newDf = dfManager.getDataflow(modelId);
dfManager.updateDataflowDetailsLayouts(newDf.getSegments().get(0),
- newDf.getSegments().get(0).getLayoutsMap().values().stream()
- .filter(layout -> layout.getLayoutId() != tobeDeleteLayoutId).collect(Collectors.toList()));
+ Collections.singletonList(tobeDeleteLayoutId), Collections.emptyList());
// remove seg2 and tobedelete layout should be cleared from indexplan
val update3 = new NDataflowUpdate(newDf.getUuid());
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
index f18ee8db22..9db76dd0df 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
@@ -639,7 +639,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
val reModel = modelManager.getDataModelDescByAlias("nmodel_basic_inner");
Assert.assertNotNull(reModel);
@@ -701,7 +701,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
val reModel = modelManager.getDataModelDescByAlias("nmodel_basic_inner");
Assert.assertNotNull(reModel);
@@ -751,7 +751,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
val reModel = modelManager.getDataModelDescByAlias("nmodel_basic_inner");
Assert.assertNotNull(reModel);
@@ -812,7 +812,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
indexPlan = NIndexPlanManager.getInstance(getTestConfig(), PROJECT).getIndexPlan(originModel.getUuid());
Assert.assertEquals(1, indexPlan.getRuleBasedIndex().getAggregationGroups().size());
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index 14b835de6c..baffa9d374 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -612,9 +612,7 @@ public class TableServiceTest extends CSVSourceTestCase {
NProjectManager npr = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
NTableMetadataManager tableManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
"default");
- ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase(removeDB);
- npr.updateProject(projectInstance);
+ npr.updateProject("default", copyForWrite -> copyForWrite.setDefaultDatabase(removeDB));
Assert.assertEquals(removeDB, npr.getDefaultDatabase("default"));
for (TableDesc table : tableManager.listAllTables()) {
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7b6af176aa..60837454a0 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -75,6 +75,7 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.common.util.JsonUtil;
@@ -360,7 +361,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
public void saveQuery(final String creator, final String project, final Query query) throws IOException {
aclEvaluate.checkProjectQueryPermission(project);
Message msg = MsgPicker.getMsg();
- val record = getSavedQueries(creator, project);
+ val record = getSavedQueries(creator, project, true);
+ assert record != null;
List<Query> currentQueries = record.getQueries();
if (currentQueries.stream().map(Query::getName).collect(Collectors.toSet()).contains(query.getName()))
throw new KylinException(SAVE_QUERY_FAILED,
@@ -373,18 +375,31 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
@Transaction(project = 1)
public void removeSavedQuery(final String creator, final String project, final String id) throws IOException {
aclEvaluate.checkProjectQueryPermission(project);
- val record = getSavedQueries(creator, project);
+ val record = getSavedQueries(creator, project, true);
+ assert record != null;
record.setQueries(record.getQueries().stream().filter(q -> !q.getId().equals(id)).collect(Collectors.toList()));
getStore().checkAndPutResource(getQueryKeyById(project, creator), record, QueryRecordSerializer.getInstance());
}
public QueryRecord getSavedQueries(final String creator, final String project) throws IOException {
+ return getSavedQueries(creator, project, false);
+ }
+
+ private QueryRecord getSavedQueries(final String creator, final String project, boolean withWriteLock)
+ throws IOException {
aclEvaluate.checkProjectQueryPermission(project);
if (null == creator) {
return null;
}
- QueryRecord record = getStore().getResource(getQueryKeyById(project, creator),
- QueryRecordSerializer.getInstance());
+ String resPath = getQueryKeyById(project, creator);
+ ResourceStore resourceStore = getStore();
+ QueryRecord record;
+ if (withWriteLock) {
+ record = MemoryLockUtils.doWithLock(null, resPath, false, resourceStore,
+ () -> resourceStore.getResource(resPath, QueryRecordSerializer.getInstance()));
+ } else {
+ record = getStore().getResource(resPath, QueryRecordSerializer.getInstance());
+ }
if (record == null) {
return new QueryRecord();
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 5209745448..200fbb960f 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -94,6 +94,7 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.NativeQueryRealization;
@@ -1240,6 +1241,27 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
}
}
+ @Test
+ public void testSaveAndRemoveQueryWithTransaction() throws IOException {
+ Query query = new Query("test", "default", "test_sql", "test_description");
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ queryService.saveQuery("admin", "default", query);
+ return null;
+ }, "default", 1);
+
+ QueryService.QueryRecord queryRecord = queryService.getSavedQueries("admin", "default");
+ Assert.assertEquals(1, queryRecord.getQueries().size());
+ Assert.assertEquals("test", queryRecord.getQueries().get(0).getName());
+
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ queryService.removeSavedQuery("admin", "default", query.getId());
+ return null;
+ }, "default", 1);
+
+ queryRecord = queryService.getSavedQueries("admin", "default");
+ Assert.assertTrue(queryRecord.getQueries().isEmpty());
+ }
+
@Test
public void testCacheSignature() {
val project = "default";
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java
index a0f384931d..02445a2936 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java
@@ -32,7 +32,6 @@ import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryMetricsContext;
import org.apache.kylin.rest.response.SQLResponse;
@@ -173,8 +172,9 @@ public class QueryCacheSignatureUtilTest extends NLocalFileMetadataTestCase {
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- TableDesc table = tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
- table.setLastModified(System.currentTimeMillis());
+ tableMetadataManager.updateTableDesc("DEFAULT.TEST_ORDER", copyForWrite -> {
+ copyForWrite.setLastModified(System.currentTimeMillis());
+ });
Assert.assertTrue(QueryCacheSignatureUtil.checkCacheExpired(sqlResponse, project));
}
@@ -194,8 +194,9 @@ public class QueryCacheSignatureUtilTest extends NLocalFileMetadataTestCase {
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- TableDesc table = tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
- table.setLastModified(System.currentTimeMillis());
+ tableMetadataManager.updateTableDesc("DEFAULT.TEST_ORDER", copyForWrite -> {
+ copyForWrite.setLastModified(System.currentTimeMillis());
+ });
Assert.assertTrue(QueryCacheSignatureUtil.checkCacheExpired(sqlResponse, project));
}
@@ -217,8 +218,9 @@ public class QueryCacheSignatureUtilTest extends NLocalFileMetadataTestCase {
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- TableDesc table = tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
- table.setLastModified(System.currentTimeMillis());
+ tableMetadataManager.updateTableDesc("DEFAULT.TEST_ORDER", copyForWrite -> {
+ copyForWrite.setLastModified(System.currentTimeMillis());
+ });
Assert.assertTrue(QueryCacheSignatureUtil.checkCacheExpired(sqlResponse, project));
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java
index 2f2dc57d29..33be350e1b 100644
--- a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java
+++ b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java
@@ -24,10 +24,11 @@ import java.io.Serializable;
import java.util.List;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Data;
@@ -40,6 +41,11 @@ public class SQLBlacklist extends RootPersistentEntity implements Serializable {
public SQLBlacklist() {
updateRandomUuid();
}
+
+ public SQLBlacklist(String project) {
+ this();
+ this.project = project;
+ }
@JsonProperty("project")
private String project;
@@ -135,4 +141,9 @@ public class SQLBlacklist extends RootPersistentEntity implements Serializable {
public String resourceName() {
return project;
}
+
+ @Override
+ public String getResourcePath() {
+ return SQL_BLACKLIST_RESOURCE_ROOT + "/" + resourceName() + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java
index e37d492112..e20ab84c1f 100644
--- a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java
+++ b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java
@@ -22,14 +22,13 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
public class SQLBlacklistManager {
private static final Logger logger = LoggerFactory.getLogger(SQLBlacklistManager.class);
@@ -102,23 +101,28 @@ public class SQLBlacklistManager {
public SQLBlacklist getSqlBlacklist(String project) {
return crud.get(project);
}
+
+ private SQLBlacklist copyForWrite(SQLBlacklist sqlBlacklist) {
+ return crud.copyForWrite(sqlBlacklist);
+ }
public SQLBlacklist saveSqlBlacklist(SQLBlacklist sqlBlacklist) throws IOException {
SQLBlacklist savedSqlBlacklist = getSqlBlacklist(sqlBlacklist.getProject());
+ SQLBlacklist copy;
if (null != savedSqlBlacklist) {
- savedSqlBlacklist.setBlacklistItems(sqlBlacklist.getBlacklistItems());
- crud.save(savedSqlBlacklist);
+ copy = copyForWrite(savedSqlBlacklist);
+ copy.setBlacklistItems(sqlBlacklist.getBlacklistItems());
} else {
- crud.save(sqlBlacklist);
+ copy = copyForWrite(sqlBlacklist);
}
+ crud.save(copy);
return sqlBlacklist;
}
public SQLBlacklist addSqlBlacklistItem(String project, SQLBlacklistItem sqlBlacklistItem) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
- sqlBlacklist = new SQLBlacklist();
- sqlBlacklist.setProject(project);
+ sqlBlacklist = copyForWrite(new SQLBlacklist(project));
}
sqlBlacklist.addBlacklistItem(sqlBlacklistItem);
crud.save(sqlBlacklist);
@@ -126,7 +130,7 @@ public class SQLBlacklistManager {
}
public SQLBlacklist updateSqlBlacklistItem(String project, SQLBlacklistItem sqlBlacklistItem) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
return null;
}
@@ -142,7 +146,7 @@ public class SQLBlacklistManager {
}
public SQLBlacklist deleteSqlBlacklistItem(String project, String id) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
return null;
}
@@ -152,7 +156,7 @@ public class SQLBlacklistManager {
}
public SQLBlacklist clearBlacklist(String project) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
return null;
}
diff --git a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
index 808f1537b0..0dd1776bcd 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
@@ -108,7 +108,7 @@ public class MetaStoreHealthIndicator implements HealthIndicator, ApplicationLis
@VisibleForTesting
public Health allNodeCheck() {
return UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.<Health> builder().skipAuditLog(true)
- .unitName(UNIT_NAME).maxRetry(MAX_RETRY).processor(() -> {
+ .unitName(UNIT_NAME).tempLockName(UNIT_NAME).maxRetry(MAX_RETRY).processor(() -> {
ResourceStore store;
try {
store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
@@ -126,6 +126,7 @@ public class MetaStoreHealthIndicator implements HealthIndicator, ApplicationLis
logger.trace(op);
start = System.currentTimeMillis();
try {
+
store.checkAndPutResource(resourcePath, new StringEntity(uuid), StringEntity.serializer);
checkTime(start, op);
} catch (Exception e) {
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index 5e21a23224..6b3d4d5641 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -142,7 +142,11 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
val scheduler = NDefaultScheduler.getInstance(getProject());
Optional.ofNullable(scheduler.getContext()).ifPresent(context -> {
Optional.ofNullable(context.getRunningJobThread(this)).ifPresent(taskThread -> {
- taskThread.interrupt();
+ // workaround for https://olapio.atlassian.net/browse/KE-41836
+ if (Arrays.stream(taskThread.getStackTrace())
+ .noneMatch(stack -> stack.getClassName().contains("FetcherRunner"))) {
+ taskThread.interrupt();
+ }
context.removeRunningJob(this);
});
});
@@ -522,16 +526,18 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
ResourceStore.dumpKylinProps(tmpDir, props);
} else {
// The way of Updating metadata is CopyOnWrite. So it is safe to use Reference in the value.
- Map<String, RawResource> dumpMap = EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(
- UnitOfWorkParams.<Map> builder().readonly(true).unitName(getProject()).maxRetry(1).processor(() -> {
- Map<String, RawResource> retMap = Maps.newHashMap();
- for (String resPath : getMetadataDumpList(config)) {
- ResourceStore resourceStore = ResourceStore.getKylinMetaStore(config);
- RawResource rawResource = resourceStore.getResource(resPath);
- retMap.put(resPath, rawResource);
- }
- return retMap;
- }).build());
+ Set<String> metadataDumpList = getMetadataDumpList(config);
+ Map<String, RawResource> dumpMap = EnhancedUnitOfWork
+ .doInTransactionWithCheckAndRetry(UnitOfWorkParams.<Map<String, RawResource>> builder()
+ .readonly(true).unitName(getProject()).maxRetry(1).processor(() -> {
+ Map<String, RawResource> retMap = Maps.newHashMap();
+ for (String resPath : metadataDumpList) {
+ ResourceStore resourceStore = ResourceStore.getKylinMetaStore(config);
+ RawResource rawResource = resourceStore.getResource(resPath);
+ retMap.put(resPath, rawResource);
+ }
+ return retMap;
+ }).build());
if (Objects.isNull(dumpMap) || dumpMap.isEmpty()) {
return;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java
index 314ca23805..fb8684bbd6 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.spark.job;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.val;
import org.apache.hadoop.fs.FileStatus;
@@ -61,10 +62,11 @@ public class NSparkUpdateMetadataStep extends AbstractExecutable {
Preconditions.checkArgument(parent instanceof DefaultExecutableOnModel);
val handler = ((DefaultExecutableOnModel) parent).getHandler();
try {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(UnitOfWorkParams.builder().processor(() -> {
handler.handleFinished();
return null;
- }, context.getEpochId(), handler.getProject());
+ }).retryMoreTimeForDeadLockException(true).epochId(context.getEpochId()).unitName(handler.getProject())
+ .build());
cleanExpiredSnapshot();
return ExecuteResult.createSucceed();
} catch (Throwable throwable) {
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala
index 31eb13f7d7..ffac6bf50f 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.builder
-import org.apache.kylin.common.persistence.transaction.UnitOfWork
+import org.apache.kylin.common.persistence.transaction.{UnitOfWork, UnitOfWorkParams}
import org.apache.kylin.common.{KapConfig, KylinConfig}
import org.apache.kylin.engine.spark.utils.LogUtils
import org.apache.kylin.metadata.model.NTableMetadataManager
@@ -62,7 +62,9 @@ class SnapshotPartitionBuilder extends SnapshotBuilder {
copyTable
}
}
- UnitOfWork.doInTransactionWithRetry(new TableUpdateOps, project)
+ val params = UnitOfWorkParams.builder.unitName(project).maxRetry(3).useProjectLock(true)
+ .processor((new TableUpdateOps).asInstanceOf[UnitOfWork.Callback[Nothing]]).build()
+ UnitOfWork.doInTransactionWithRetry(params)
log.info(s"check point partitions for $tableName , partition $partition")
}
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
index 860ed0ad7d..d999411b4c 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
@@ -423,10 +423,11 @@ public class JobManagerTest extends NLocalFileMetadataTestCase {
val segmentRange = new SegmentRange.TimePartitionedSegmentRange(startTime, endTime);
val df = dfm.getDataflow(modelId);
val newSegment = dfm.appendSegment(df, segmentRange, status, partitions);
- newSegment.getMultiPartitions().forEach(partition -> {
+ val copySegment = dfm.copyForWrite(df).getSegment(newSegment.getId());
+ copySegment.getMultiPartitions().forEach(partition -> {
partition.setStatus(PartitionStatusEnum.READY);
});
- return newSegment;
+ return copySegment;
}
private NDataLayout generateLayoutForMultiPartition(String modelId, String segmentId, List<String> partitionValues,
diff --git a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
index d6f4da6cbf..559e3d808e 100644
--- a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
+++ b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
@@ -28,6 +28,8 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.TempMetadataBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -36,9 +38,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.val;
public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
@@ -113,15 +112,16 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
public void testPushdownJdbc() throws Exception {
Connection h2Connection = DriverManager.getConnection("jdbc:h2:mem:db_default", "sa", "");
NProjectManager npr = NProjectManager.getInstance(getTestConfig());
+ npr.updateProject("default", copyForWrite -> {
+ copyForWrite.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ copyForWrite.setOverrideKylinProps(overrideKylinProps);
+ });
ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase("SSB");
- LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
- overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
- projectInstance.setOverrideKylinProps(overrideKylinProps);
- npr.updateProject(projectInstance);
KylinConfigExt config = projectInstance.getConfig();
PushDownRunnerJdbcImpl pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
pushDownRunnerJdbc.init(config, projectInstance.getName());
@@ -135,15 +135,16 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
@Test
public void testProjectPushDownJdbc() throws Exception {
NProjectManager npr = NProjectManager.getInstance(getTestConfig());
+ npr.updateProject("default", copyForWrite -> {
+ copyForWrite.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ copyForWrite.setOverrideKylinProps(overrideKylinProps);
+ });
ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase("SSB");
- LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
- overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
- projectInstance.setOverrideKylinProps(overrideKylinProps);
- npr.updateProject(projectInstance);
KylinConfigExt config = projectInstance.getConfig();
PushDownRunnerJdbcImpl pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
pushDownRunnerJdbc.init(config, projectInstance.getName());
@@ -153,15 +154,16 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
pushDownRunnerJdbc.executeQuery(sql, returnRows, returnColumnMeta, "default");
Assert.assertEquals("1", returnRows.get(0).get(0));
+ npr.updateProject("demo", copyForWrite -> {
+ copyForWrite.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ copyForWrite.setOverrideKylinProps(overrideKylinProps);
+ });
ProjectInstance projectInstance2 = npr.getProject("demo");
- projectInstance2.setDefaultDatabase("SSB");
- overrideKylinProps = Maps.newLinkedHashMap();
- overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
- projectInstance2.setOverrideKylinProps(overrideKylinProps);
- npr.updateProject(projectInstance2);
config = projectInstance2.getConfig();
pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
pushDownRunnerJdbc.init(config, projectInstance2.getName());
diff --git a/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java b/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
index 777a4830f5..728c816c9e 100644
--- a/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
+++ b/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
@@ -77,19 +77,24 @@ public class StreamingJobManager {
public void createStreamingJob(NDataModel model, JobTypeEnum jobType) {
StreamingJobMeta job = StreamingJobMeta.create(model, JobStatusEnum.STOPPED, jobType);
- crud.save(job);
+ StreamingJobMeta copy = copyForWrite(job);
+ crud.save(copy);
}
public StreamingJobMeta copy(StreamingJobMeta jobMeta) {
return crud.copyBySerialization(jobMeta);
}
+
+ public StreamingJobMeta copyForWrite(StreamingJobMeta jobMeta) {
+ return crud.copyForWrite(jobMeta);
+ }
public StreamingJobMeta updateStreamingJob(String jobId, NStreamingJobUpdater updater) {
StreamingJobMeta cached = getStreamingJobByUuid(jobId);
if (cached == null) {
return null;
}
- StreamingJobMeta copy = copy(cached);
+ StreamingJobMeta copy = copyForWrite(cached);
updater.modify(copy);
return crud.save(copy);
}
diff --git a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index f3d920d918..d7b1da467e 100644
--- a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
+++ b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -311,8 +311,11 @@ public class MetadataToolHelper {
val projectName = Paths.get(projectPath).getName(0).toString();
val destResources = currentResourceStore.listResourcesRecursively(projectPath);
val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
- UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
- destResources, srcResources, delete), projectName, 1);
+ UnitOfWorkParams<Object> params = UnitOfWorkParams.builder().unitName(projectName).maxRetry(1)
+ .useProjectLock(true).processor(() -> doRestore(currentResourceStore, restoreResourceStore,
+ destResources, srcResources, delete))
+ .build();
+ UnitOfWork.doInTransactionWithRetry(params);
}
} else {
@@ -332,16 +335,20 @@ public class MetadataToolHelper {
Set<String> finalGlobalDestResources = globalDestResources;
- UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
- finalGlobalDestResources, globalSrcResources, delete), UnitOfWork.GLOBAL_UNIT, 1);
+ UnitOfWorkParams<Object> params = UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT).maxRetry(1)
+ .useProjectLock(true).processor(() -> doRestore(currentResourceStore, restoreResourceStore,
+ finalGlobalDestResources, globalSrcResources, delete))
+ .build();
+ UnitOfWork.doInTransactionWithRetry(params);
val projectPath = FileSystems.getDefault().getSeparator() + project;
val destResources = currentResourceStore.listResourcesRecursively(projectPath);
val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
- UnitOfWork.doInTransactionWithRetry(
- () -> doRestore(currentResourceStore, restoreResourceStore, destResources, srcResources, delete),
- project, 1);
+ params = UnitOfWorkParams.builder().unitName(project).maxRetry(1).useProjectLock(true).processor(
+ () -> doRestore(currentResourceStore, restoreResourceStore, destResources, srcResources, delete))
+ .build();
+ UnitOfWork.doInTransactionWithRetry(params);
}
logger.info("restore successfully");
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java b/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
index 0e6ab4a14c..5e507becea 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
@@ -51,6 +51,7 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.TrashRecord;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.CliCommandExecutor.CliCmdExecResult;
@@ -200,13 +201,10 @@ public class StorageCleaner implements GarbageCleaner {
}
log.debug("folder {} is collected，detailed -> {}", allFileSystem.getPath(), allFileSystems);
}
- UnitOfWork.doInTransactionWithRetry(() -> {
- collectDeletedProject();
- for (ProjectInstance project : projects) {
- collect(project.getName());
- }
- return null;
- }, UnitOfWork.GLOBAL_UNIT);
+ collectDeletedProject();
+ for (ProjectInstance project : projects) {
+ collect(project.getName());
+ }
long configSurvivalTimeThreshold = timeMachineEnabled ? kylinConfig.getStorageResourceSurvivalTimeThreshold()
: config.getCuboidLayoutSurvivalTimeThreshold();
@@ -307,6 +305,9 @@ public class StorageCleaner implements GarbageCleaner {
ResourceStore threadViewRS = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
RawResource raw = resourceStore.getResource(ResourceStore.METASTORE_TRASH_RECORD);
long mvcc = raw == null ? -1 : raw.getMvcc();
+ // TrashRecord doesn't extend RootPersistentEntity. Let's manually lock it's resource path.
+ MemoryLockUtils.doWithLock(ResourceStore.METASTORE_TRASH_RECORD, false, threadViewRS,
+ () -> null);
threadViewRS.checkAndPutResource(ResourceStore.METASTORE_TRASH_RECORD,
ByteSource.wrap(JsonUtil.writeValueAsBytes(new TrashRecord(trashRecord))), mvcc);
return 0;
@@ -485,13 +486,14 @@ public class StorageCleaner implements GarbageCleaner {
String result = "";
try {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- Set<String> jobTempTables = jobTemps.stream()
- .map(node -> tableCleanerHelper.getJobTransactionalTable(project, node.getName()))
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
+ Set<String> jobTempTables = jobTemps.stream().filter(node -> !node.getName().endsWith(".zip"))
+ .map(node -> tableCleanerHelper.getJobTransactionalTable(project, node.getName(), fs))
.flatMap(Collection::stream).collect(Collectors.toSet());
Set<String> discardTempTables = NExecutableManager.getInstance(config, project)
.getExecutablesByStatus(ExecutableState.DISCARDED).stream()
- .map(e -> tableCleanerHelper.getJobTransactionalTable(project, e.getId()))
+ .map(e -> tableCleanerHelper.getJobTransactionalTable(project, e.getId(), fs))
.flatMap(Collection::stream).collect(Collectors.toSet());
jobTempTables.addAll(discardTempTables);
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java
index 7c186cfdde..3d5b167096 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java
@@ -23,6 +23,7 @@ import java.util.Locale;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.util.JsonUtil;
... 208 lines suppressed ...
(kylin) 05/35: KYLIN-5797 Remove nacos jar
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 82ee25fdf1a66601a3cebcab4ca86c1664da57c4
Author: Zhimin Wu <59...@qq.com>
AuthorDate: Wed Aug 30 14:00:26 2023 +0800
KYLIN-5797 Remove nacos jar
---
src/server/pom.xml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/server/pom.xml b/src/server/pom.xml
index deb02b3ade..5f96ac4bc7 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -420,6 +420,8 @@
org.json4s,
org.mortbay.jetty,
org.scala-lang,
+ com.alibaba.nacos,
+ com.alibaba.cloud,
com.google.flatbuffers,
com.google.inject
com.sun.jersey,
(kylin) 18/35: KYLIN-5814 Support lettuce sentinel mode
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ebd2f3d6704a566896b60f7a3c67bf7226c67348
Author: haocheni <ha...@qq.com>
AuthorDate: Thu Sep 14 10:00:44 2023 +0800
KYLIN-5814 Support lettuce sentinel mode
---
pom.xml | 7 +
src/common-service/pom.xml | 4 +
.../kylin/rest/cache/AbstractKylinCache.java | 78 +++++++
.../org/apache/kylin/rest/cache/CacheConstant.java | 29 +++
.../org/apache/kylin/rest/cache/RedisCache.java | 91 ++-------
.../org/apache/kylin/rest/cache/RedisCacheV2.java | 224 +++++++++++++++++++++
.../apache/kylin/rest/cache/CacheConstantTest.java | 30 +++
.../apache/kylin/rest/cache/RedisCacheTest.java | 36 ++++
.../apache/kylin/rest/cache/RedisCacheV2Test.java | 195 ++++++++++++++++++
.../org/apache/kylin/common/KylinConfigBase.java | 8 +
.../kylin/common/exception/ServerErrorCode.java | 1 +
.../resources/kylin_errorcode_conf_en.properties | 1 +
.../resources/kylin_errorcode_conf_zh.properties | 1 +
.../kylin/metadata/query/QueryMetricsContext.java | 3 +-
src/query-service/pom.xml | 5 +
.../kylin/rest/service/QueryCacheManager.java | 113 +++++++++--
.../kylin/rest/service/QueryCacheManagerTest.java | 191 ++++++++++++++++++
17 files changed, 923 insertions(+), 94 deletions(-)
diff --git a/pom.xml b/pom.xml
index 88261b4d28..07224bd0f1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -188,6 +188,7 @@
<ehcache.version>2.10.9.2</ehcache.version>
<net.spy.memcached.verion>2.12.3</net.spy.memcached.verion>
<redis.version>3.8.0</redis.version>
+ <lettuce.version>6.1.8.RELEASE</lettuce.version>
<apache-httpclient.version>4.5.13</apache-httpclient.version>
<beanutils.version>1.9.4</beanutils.version>
<xerces.version>2.12.2</xerces.version>
@@ -797,6 +798,12 @@
<version>${redis.version}</version>
</dependency>
+ <dependency>
+ <groupId>io.lettuce</groupId>
+ <artifactId>lettuce-core</artifactId>
+ <version>${lettuce.version}</version>
+ </dependency>
+
<!--KAP test scope-->
<dependency>
<groupId>org.apache.kylin</groupId>
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index 747735c9ea..7a0d6d3403 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -75,6 +75,10 @@
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.lettuce</groupId>
+ <artifactId>lettuce-core</artifactId>
+ </dependency>
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java b/src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java
new file mode 100644
index 0000000000..9461a29f09
--- /dev/null
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+import static org.apache.kylin.rest.cache.CacheConstant.PREFIX;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.rest.service.CommonQueryCacheSupporter;
+import org.apache.kylin.rest.util.SerializeUtil;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public abstract class AbstractKylinCache implements KylinCache {
+ protected boolean isExceptionQuery(String type) {
+ return type.equals(CommonQueryCacheSupporter.Type.EXCEPTION_QUERY_CACHE.rootCacheName);
+ }
+
+ protected String getTypeProjectPrefix(String type, String project) {
+ return String.format(Locale.ROOT, "%s-%s", type, project);
+ }
+
+ protected byte[] convertKeyToByte(String type, Object key) {
+ try {
+ String prefixAndType = PREFIX + type;
+ byte[] typeBytes = getBytesFromString(prefixAndType);
+ byte[] keyBytes = SerializeUtil.serialize(key);
+ byte[] trueKeyBytes = new byte[keyBytes.length + typeBytes.length];
+ System.arraycopy(typeBytes, 0, trueKeyBytes, 0, typeBytes.length);
+ System.arraycopy(keyBytes, 0, trueKeyBytes, typeBytes.length, keyBytes.length);
+ return trueKeyBytes;
+ } catch (Exception e) {
+ log.error("serialize fail!", e);
+ return new byte[0];
+ }
+ }
+
+ protected byte[] convertValueToByte(Object value) {
+ try {
+ return CompressionUtils.compress(SerializeUtil.serialize(value));
+ } catch (Exception e) {
+ log.error("serialize failed!", e);
+ return new byte[0];
+ }
+ }
+
+ protected byte[] getBytesFromString(String str) {
+ return str.getBytes(StandardCharsets.UTF_8);
+ }
+
+ protected Object convertByteToObject(byte[] bytes) {
+ try {
+ return SerializeUtil.deserialize(CompressionUtils.decompress(bytes));
+ } catch (Exception e) {
+ log.error("deserialize fail!", e);
+ return null;
+ }
+ }
+}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/cache/CacheConstant.java b/src/common-service/src/main/java/org/apache/kylin/rest/cache/CacheConstant.java
new file mode 100644
index 0000000000..53544743d8
--- /dev/null
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/cache/CacheConstant.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+public class CacheConstant {
+ public static final String NX = "NX";
+ public static final String XX = "XX";
+ public static final String PREFIX = "Kylin-";
+
+ private CacheConstant() {
+ throw new IllegalStateException("Utility class");
+ }
+}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCache.java b/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCache.java
index cf49508eb3..5ef1d8568d 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCache.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCache.java
@@ -18,22 +18,32 @@
package org.apache.kylin.rest.cache;
-import org.apache.kylin.guava30.shaded.common.base.Joiner;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import static org.apache.kylin.rest.cache.CacheConstant.NX;
+import static org.apache.kylin.rest.cache.CacheConstant.PREFIX;
+import static org.apache.kylin.rest.cache.CacheConstant.XX;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.Singletons;
-import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.EncryptUtil;
-import org.apache.kylin.rest.service.CommonQueryCacheSupporter;
-import org.apache.kylin.rest.util.SerializeUtil;
+import org.apache.kylin.guava30.shaded.common.base.Joiner;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisCluster;
@@ -48,24 +58,10 @@ import redis.clients.jedis.exceptions.JedisMovedDataException;
import redis.clients.jedis.params.SetParams;
import redis.clients.jedis.util.JedisClusterCRC16;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class RedisCache implements KylinCache {
+public class RedisCache extends AbstractKylinCache implements KylinCache {
private static final Logger logger = LoggerFactory.getLogger(RedisCache.class);
- private static final String NX = "NX";
- private static final String XX = "XX";
- private static final String PREFIX = "Kylin5-";
- private static final String CHARSET_NAME = "UTF-8";
private static final Charset CHARSET = StandardCharsets.UTF_8;
private static final String SCAN_POINTER_START_STR = new String(ScanParams.SCAN_POINTER_START_BINARY, CHARSET);
private static JedisPool jedisPool;
@@ -234,14 +230,6 @@ public class RedisCache implements KylinCache {
return cache;
}
- private String getTypeProjectPrefix(String type, String project) {
- return String.format(Locale.ROOT, "%s-%s", type, project);
- }
-
- private boolean isExceptionQuery(String type) {
- return type.equals(CommonQueryCacheSupporter.Type.EXCEPTION_QUERY_CACHE.rootCacheName);
- }
-
@Override
public void put(String type, String project, Object key, Object value) {
if (isRecovering.get()) {
@@ -457,47 +445,6 @@ public class RedisCache implements KylinCache {
return keys;
}
- private byte[] convertValueToByte(Object value) {
- try {
- return CompressionUtils.compress(SerializeUtil.serialize(value));
- } catch (Exception e) {
- logger.error("serialize failed!", e);
- return null;
- }
- }
-
- private byte[] getBytesFromString(String str) {
- try {
- return str.getBytes(CHARSET_NAME);
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("unsupported encoding:" + CHARSET_NAME, e);
- }
- }
-
- private byte[] convertKeyToByte(String type, Object key) {
- try {
- String prefixAndType = PREFIX + type;
- byte[] typeBytes = getBytesFromString(prefixAndType);
- byte[] keyBytes = SerializeUtil.serialize(key);
- byte[] trueKeyBytes = new byte[keyBytes.length + typeBytes.length];
- System.arraycopy(typeBytes, 0, trueKeyBytes, 0, typeBytes.length);
- System.arraycopy(keyBytes, 0, trueKeyBytes, typeBytes.length, keyBytes.length);
- return trueKeyBytes;
- } catch (Exception e) {
- logger.error("serialize fail!", e);
- return null;
- }
- }
-
- private Object convertByteToObject(byte[] bytes) {
- try {
- return SerializeUtil.deserialize(CompressionUtils.decompress(bytes));
- } catch (Exception e) {
- logger.error("deserialize fail!", e);
- return null;
- }
- }
-
@Override
public boolean remove(String type, String project, Object key) {
if (isRecovering.get()) {
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCacheV2.java b/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCacheV2.java
new file mode 100644
index 0000000000..99fe1cd50e
--- /dev/null
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/cache/RedisCacheV2.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+import static org.apache.kylin.common.exception.ServerErrorCode.REDIS_INIT_FAILED;
+import static org.apache.kylin.rest.cache.CacheConstant.NX;
+import static org.apache.kylin.rest.cache.CacheConstant.PREFIX;
+import static org.apache.kylin.rest.cache.CacheConstant.XX;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.Singletons;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.util.EncryptUtil;
+
+import io.lettuce.core.KeyScanCursor;
+import io.lettuce.core.ReadFrom;
+import io.lettuce.core.RedisClient;
+import io.lettuce.core.RedisURI;
+import io.lettuce.core.ScanArgs;
+import io.lettuce.core.SetArgs;
+import io.lettuce.core.api.sync.RedisCommands;
+import io.lettuce.core.codec.ByteArrayCodec;
+import io.lettuce.core.internal.HostAndPort;
+import io.lettuce.core.masterreplica.MasterReplica;
+import io.lettuce.core.masterreplica.StatefulRedisMasterReplicaConnection;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class RedisCacheV2 extends AbstractKylinCache implements KylinCache {
+ private static final AtomicBoolean isRecovering = new AtomicBoolean(false);
+ private String redisExpireTimeUnit;
+ private long redisExpireTime;
+ private long redisExpireTimeForException;
+ private StatefulRedisMasterReplicaConnection<byte[], byte[]> redisConnection;
+ private RedisCommands<byte[], byte[]> syncCommands;
+
+ private RedisCacheV2() {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ loadConfigurations(kylinConfig);
+ createRedisClient(kylinConfig);
+ log.info("Redis init success.");
+ }
+
+ public static KylinCache getInstance() {
+ try {
+ return Singletons.getInstance(RedisCacheV2.class);
+ } catch (Exception e) {
+ log.error("Redis init failed:{} ", ExceptionUtils.getStackTrace(e));
+ }
+ return null;
+ }
+
+ @Override
+ public void put(String type, String project, Object key, Object value) {
+ long expireTime = isExceptionQuery(type) ? redisExpireTimeForException : redisExpireTime;
+ put(getTypeProjectPrefix(type, project), key, value, NX, redisExpireTimeUnit, expireTime);
+ }
+
+ public void put(String type, Object key, Object value, String ifExist, String expireTimeUnit, long expireTime) {
+ byte[] realKey = convertKeyToByte(type, key);
+ byte[] valueBytes = convertValueToByte(value);
+ syncCommands.set(realKey, valueBytes, createSetArgs(ifExist, expireTimeUnit, expireTime));
+ }
+
+ @Override
+ public void update(String type, String project, Object key, Object value) {
+ long expireTime = isExceptionQuery(type) ? redisExpireTimeForException : redisExpireTime;
+ put(getTypeProjectPrefix(type, project), key, value, XX, redisExpireTimeUnit, expireTime);
+ }
+
+ @Override
+ public Object get(String type, String project, Object key) {
+ byte[] realKey = convertKeyToByte(getTypeProjectPrefix(type, project), key);
+ log.trace("redis get start");
+ byte[] sqlResp = syncCommands.get(realKey);
+ log.trace("redis get done, size = {}bytes", sqlResp == null ? 0 : sqlResp.length);
+
+ if (sqlResp != null) {
+ Object result = convertByteToObject(sqlResp);
+ log.trace("redis result deserialized");
+ return result;
+ }
+ return null;
+ }
+
+ @Override
+ public boolean remove(String type, String project, Object key) {
+ Long removedCount = syncCommands.del(convertKeyToByte(getTypeProjectPrefix(type, project), key));
+ return removedCount != null && removedCount > 0;
+ }
+
+ @Override
+ public void clearAll() {
+ clearByType("", "");
+ }
+
+ @Override
+ public void clearByType(String type, String project) {
+ String prefixAndType = PREFIX;
+ if (!type.isEmpty()) {
+ prefixAndType += getTypeProjectPrefix(type, project);
+ }
+ KeyScanCursor<byte[]> scanCursor = syncCommands.scan(ScanArgs.Builder.matches(prefixAndType + "*"));
+
+ if (scanCursor.isFinished()) {
+ for (byte[] key : scanCursor.getKeys()) {
+ syncCommands.del(key);
+ }
+ }
+ while (!scanCursor.isFinished()) {
+ for (byte[] key : scanCursor.getKeys()) {
+ syncCommands.del(key);
+ }
+ scanCursor = syncCommands.scan(scanCursor);
+ }
+ }
+
+ public KylinCache recoverInstance() {
+ KylinCache kylinCache = this;
+ if (isRecovering.compareAndSet(false, true)) {
+ try {
+ log.info("Destroy RedisCacheV2.");
+ if (redisConnection != null) {
+ redisConnection.close();
+ }
+ Singletons.clearInstance(RedisCacheV2.class);
+ log.info("Initiate RedisCacheV2.");
+ kylinCache = Singletons.getInstance(RedisCacheV2.class);
+ } finally {
+ isRecovering.set(false);
+ }
+ }
+ return kylinCache;
+ }
+
+ private void loadConfigurations(KylinConfig kylinConfig) {
+ redisExpireTimeUnit = kylinConfig.getRedisExpireTimeUnit();
+ redisExpireTime = kylinConfig.getRedisExpireTime();
+ redisExpireTimeForException = kylinConfig.getRedisExpireTimeForException();
+ }
+
+ private List<HostAndPort> parseHosts(KylinConfig kylinConfig) {
+ String redisHosts = kylinConfig.getRedisHosts().trim();
+ String[] hostAndPorts = redisHosts.split(",");
+ if (Arrays.stream(hostAndPorts).anyMatch(StringUtils::isBlank)) {
+ throw new KylinException(REDIS_INIT_FAILED, "Redis client init failed because there are "
+ + "some errors in kylin.properties for 'kylin.cache.redis.hosts'");
+ }
+
+ return Arrays.stream(hostAndPorts).map(hostAndPort -> {
+ String host = hostAndPort.split(":")[0].trim();
+ int port = Integer.parseInt(hostAndPort.split(":")[1]);
+ return HostAndPort.of(host, port);
+ }).collect(Collectors.toList());
+ }
+
+ private char[] parsePassword(KylinConfig kylinConfig) {
+ String redisPassword = kylinConfig.getRedisPassword();
+ if (EncryptUtil.isEncrypted(redisPassword)) {
+ redisPassword = EncryptUtil.getDecryptedValue(redisPassword);
+ }
+ return redisPassword == null ? null : redisPassword.toCharArray();
+ }
+
+ private void createRedisClient(KylinConfig kylinConfig) {
+ List<HostAndPort> hostAndPorts = parseHosts(kylinConfig);
+ char[] redisPassword = parsePassword(kylinConfig);
+
+ log.info("The 'kylin.cache.redis.sentinel-enabled' is {}", kylinConfig.isRedisSentinelEnabled());
+ if (kylinConfig.isRedisSentinelEnabled()) {
+ log.info("kylin will use redis sentinel");
+ RedisURI redisURI = buildRedisURI(kylinConfig, hostAndPorts, redisPassword);
+ RedisClient redisClient = RedisClient.create();
+ redisConnection = MasterReplica.connect(redisClient, ByteArrayCodec.INSTANCE, redisURI);
+ redisConnection.setReadFrom(ReadFrom.REPLICA_PREFERRED);
+ syncCommands = redisConnection.sync();
+ log.info("Lettuce sentinel ping:{}", syncCommands.ping());
+ }
+ }
+
+ private RedisURI buildRedisURI(KylinConfig kylinConfig, List<HostAndPort> hostAndPorts, char[] redisPassword) {
+ RedisURI.Builder redisUriBuilder = RedisURI.builder()
+ .withSentinelMasterId(kylinConfig.getRedisSentinelMasterId()).withPassword(redisPassword)
+ .withTimeout(Duration.ofMillis(kylinConfig.getRedisConnectionTimeout()));
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ redisUriBuilder.withSentinel(hostAndPort.getHostText(), hostAndPort.getPort());
+ }
+ RedisURI redisURI = redisUriBuilder.build();
+ log.info("Redis uri:{}", redisURI);
+ return redisURI;
+ }
+
+ private SetArgs createSetArgs(String ifExist, String expireTimeUnit, Long expireTime) {
+ SetArgs setArgs = new SetArgs();
+ setArgs = expireTimeUnit.equals("EX") ? setArgs.ex(expireTime) : setArgs.px(expireTime);
+ setArgs = ifExist.equals(NX) ? setArgs.nx() : setArgs.xx();
+ return setArgs;
+ }
+
+}
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/cache/CacheConstantTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/cache/CacheConstantTest.java
new file mode 100644
index 0000000000..89942597f7
--- /dev/null
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/cache/CacheConstantTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+import org.apache.kylin.common.Singletons;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class CacheConstantTest {
+ @Test
+ void test() {
+ Assertions.assertThrows(RuntimeException.class, () -> Singletons.getInstance(CacheConstant.class));
+ }
+}
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheTest.java
new file mode 100644
index 0000000000..dc29a70a44
--- /dev/null
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@MetadataInfo
+@ExtendWith(MockitoExtension.class)
+class RedisCacheTest {
+
+ @Test
+ void testRecoverInstance() {
+ assertDoesNotThrow(RedisCache::recoverInstance);
+ }
+}
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheV2Test.java b/src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheV2Test.java
new file mode 100644
index 0000000000..50b8cdb241
--- /dev/null
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/cache/RedisCacheV2Test.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+import static org.apache.kylin.common.util.EncryptUtil.ENC_PREFIX;
+import static org.apache.kylin.common.util.EncryptUtil.ENC_SUBFIX;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.zip.DataFormatException;
+
+import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.rest.util.SerializeUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import io.lettuce.core.KeyScanCursor;
+import io.lettuce.core.ScanArgs;
+import io.lettuce.core.api.sync.RedisCommands;
+
+@MetadataInfo
+@ExtendWith(MockitoExtension.class)
+class RedisCacheV2Test {
+ @Mock
+ private RedisCommands<byte[], byte[]> syncCommands;
+
+ @InjectMocks
+ private RedisCacheV2 redisCacheV2;
+
+ @Test
+ void testPut() {
+ String type = "type";
+ String project = "project";
+ Object key = new Object();
+ Object value = new Object();
+
+ redisCacheV2.put(type, project, key, value);
+
+ verify(syncCommands, times(1)).set(any(), any(), any());
+ }
+
+ @Test
+ void testUpdate() {
+ String type = "type";
+ String project = "project";
+ Object key = new Object();
+ Object value = new Object();
+
+ redisCacheV2.update(type, project, key, value);
+
+ verify(syncCommands, times(1)).set(any(), any(), any());
+ }
+
+ @Test
+ void testGetNull() {
+ String type = "type";
+ String project = "project";
+ Object key = new Object();
+
+ when(syncCommands.get(any())).thenReturn(null);
+
+ Object result = redisCacheV2.get(type, project, key);
+
+ assertNull(result);
+ verify(syncCommands, times(1)).get(any());
+ }
+
+ @Test
+ void testGet() throws DataFormatException, IOException {
+ String type = "type";
+ String project = "project";
+ Object key = new Object();
+ byte[] valueBytes = { -84, -19, 0, 5, 116, 0, 5, 118, 97, 108, 117, 101 };
+ when(syncCommands.get(any())).thenReturn(valueBytes);
+ Object expectedResult = SerializeUtil.deserialize(CompressionUtils.decompress(valueBytes));
+
+ Object result = redisCacheV2.get(type, project, key);
+
+ Assertions.assertEquals(expectedResult, result);
+ }
+
+ @Test
+ void testRemove() {
+ String type = "type";
+ String project = "project";
+ Object key = new Object();
+
+ when(syncCommands.del(any())).thenReturn(1L);
+
+ boolean result = redisCacheV2.remove(type, project, key);
+
+ assertTrue(result);
+ verify(syncCommands, times(1)).del(any());
+ }
+
+ @Test
+ void testClearAll() {
+ mockScanResult();
+
+ redisCacheV2.clearAll();
+
+ verify(syncCommands, atLeastOnce()).scan(any(ScanArgs.class));
+ verify(syncCommands, atLeastOnce()).del(any());
+ }
+
+ @Test
+ void testClearByType() {
+ String type = "type";
+ String project = "project";
+ mockScanResult();
+
+ redisCacheV2.clearByType(type, project);
+
+ verify(syncCommands, atLeastOnce()).scan(any(ScanArgs.class));
+ verify(syncCommands, atLeastOnce()).del(any());
+ }
+
+ @Test
+ void testGetInstance() {
+ Unsafe.overwriteSystemProp(Maps.newHashMap(), "kylin.cache.redis.sentinel-enabled", "TRUE");
+ Unsafe.overwriteSystemProp(Maps.newHashMap(), "kylin.cache.redis.sentinel-master", "default-master");
+
+ KylinCache kylinCache = RedisCacheV2.getInstance();
+ assertNull(kylinCache);
+
+ Unsafe.clearProperty("kylin.cache.redis.sentinel-enabled");
+ Unsafe.clearProperty("kylin.cache.redis.sentinel-master");
+ }
+
+ @Test
+ void testRecoverInstance() {
+ KylinCache kylinCache = redisCacheV2.recoverInstance();
+ Assertions.assertNotNull(kylinCache);
+ }
+
+ @Test
+ void testEncryptedPassword() {
+ Unsafe.overwriteSystemProp(Maps.newHashMap(), "kylin.cache.redis.password",
+ ENC_PREFIX + "password" + ENC_SUBFIX);
+ try {
+ RedisCacheV2.getInstance();
+ } catch (Exception e) {
+ Assertions.fail();
+ }
+ }
+
+ @Test
+ void testRedisHostsError() {
+ Unsafe.overwriteSystemProp(Maps.newHashMap(), "kylin.cache.redis.hosts", " ");
+
+ KylinCache kylinCache = RedisCacheV2.getInstance();
+ assertNull(kylinCache);
+
+ Unsafe.clearProperty("kylin.cache.redis.hosts");
+ }
+
+ private void mockScanResult() {
+ KeyScanCursor<byte[]> scanCursor = new KeyScanCursor<>();
+ scanCursor.setFinished(true);
+ byte[] delKeys = new byte[] { 1, 2, 3 };
+ ReflectionTestUtils.setField(scanCursor, "keys", Lists.newArrayList(delKeys));
+ when(syncCommands.scan(any(ScanArgs.class))).thenReturn(scanCursor);
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e991f4d990..a2518358a9 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1970,6 +1970,14 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.cache.redis.cluster-enabled", FALSE));
}
+ public boolean isRedisSentinelEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.cache.redis.sentinel-enabled", FALSE));
+ }
+
+ public String getRedisSentinelMasterId() {
+ return getOptional("kylin.cache.redis.sentinel-master", null);
+ }
+
public String getRedisHosts() {
return getOptional("kylin.cache.redis.hosts", "localhost:6379");
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
index 68aa6bcd3f..9b869c01a8 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
@@ -289,6 +289,7 @@ public enum ServerErrorCode implements ErrorCodeSupplier {
// 10040XXX cache
REDIS_CLEAR_ERROR("KE-010040001"), //
+ REDIS_INIT_FAILED("KE-010040002"), //
// 10050XXX SQL DDL
DDL_CHECK_ERROR("KE-010050001");
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
index 02e3f7f830..d6fd4bd8d5 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
@@ -278,5 +278,6 @@ KE-010039001=Invalid Connection Info
# cache
KE-010040001=Clear Redis Cache Failure
+KE-010040002=Redis Init Failed
# SQL DDL
KE-010050001=DDL Operation Failure
\ No newline at end of file
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
index d8fbfc42ed..fa09dca37e 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
@@ -276,5 +276,6 @@ KE-010039001=连接信息有误
# cache
KE-010040001=清空Redis缓存失败
+KE-010040002=Redis初始化失败
# SQL DDL
KE-010050001=DDL操作失败
\ No newline at end of file
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
index 341f7ecb48..3bc911a44c 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java
@@ -133,8 +133,7 @@ public class QueryMetricsContext extends QueryMetrics {
if (context.getQueryTagInfo().isHitExceptionCache() || context.getQueryTagInfo().isStorageCacheUsed()) {
this.isCacheHit = true;
- this.cacheType = KylinConfig.getInstanceFromEnv().isRedisEnabled() ? QueryHistory.CacheType.REDIS.name()
- : QueryHistory.CacheType.EHCACHE.name();
+ this.cacheType = context.getQueryTagInfo().getStorageCacheType();
}
this.resultRowCount = context.getMetrics().getResultRowCount();
this.queryMsg = context.getMetrics().getQueryMsg();
diff --git a/src/query-service/pom.xml b/src/query-service/pom.xml
index da04266338..03fb2c0072 100644
--- a/src/query-service/pom.xml
+++ b/src/query-service/pom.xml
@@ -135,6 +135,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java
index 862a7e9b8f..b809e7f210 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java
@@ -36,6 +36,7 @@ import org.apache.kylin.rest.cache.KylinCache;
import org.apache.kylin.rest.cache.KylinEhCache;
import org.apache.kylin.rest.cache.memcached.CompositeMemcachedCache;
import org.apache.kylin.rest.cache.RedisCache;
+import org.apache.kylin.rest.cache.RedisCacheV2;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.response.TableMetaCacheResult;
@@ -73,10 +74,15 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
public void init() {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
if (kylinConfig.isRedisEnabled()) {
- kylinCache = RedisCache.getInstance();
+ if (kylinConfig.isRedisSentinelEnabled()) {
+ kylinCache = RedisCacheV2.getInstance();
+ } else {
+ kylinCache = RedisCache.getInstance();
+ }
} else if (kylinConfig.isMemcachedEnabled()) {
kylinCache = CompositeMemcachedCache.getInstance();
- } else {
+ }
+ if (kylinCache == null) {
kylinCache = KylinEhCache.getInstance();
}
if (kylinCache instanceof RedisCache && checkRedisClient()) {
@@ -164,7 +170,7 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
}
public SQLResponse doSearchQuery(QueryCacheManager.Type type, SQLRequest sqlRequest) {
- Object response = kylinCache.get(type.rootCacheName, sqlRequest.getProject(), sqlRequest.getCacheKey());
+ Object response = getCache(type.rootCacheName, sqlRequest.getProject(), sqlRequest.getCacheKey());
logger.debug("[query cache log] The cache key is: {}", sqlRequest.getCacheKey());
if (response == null) {
return null;
@@ -254,7 +260,7 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
}
public TableMetaCacheResult doGetSchemaCache(String project, String userName) {
- Object metaList = kylinCache.get(Type.SCHEMA_CACHE.rootCacheName, project, userName);
+ Object metaList = getCache(Type.SCHEMA_CACHE.rootCacheName, project, userName);
if (metaList == null) {
return null;
}
@@ -286,7 +292,7 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
if (modelName != null) {
cacheKey = cacheKey + modelName;
}
- Object metaList = kylinCache.get(Type.SCHEMA_CACHE.rootCacheName, project, cacheKey);
+ Object metaList = getCache(Type.SCHEMA_CACHE.rootCacheName, project, cacheKey);
if (metaList == null) {
return null;
}
@@ -298,15 +304,15 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
if (modelName != null) {
cacheKey = cacheKey + modelName;
}
- kylinCache.put(Type.SCHEMA_CACHE.rootCacheName, project, cacheKey, schemas);
+ putCache(Type.SCHEMA_CACHE.rootCacheName, project, cacheKey, schemas);
}
public void clearSchemaCacheV2(String project, String userName) {
- kylinCache.remove(Type.SCHEMA_CACHE.rootCacheName, project, userName + "v2");
+ removeCache(Type.SCHEMA_CACHE.rootCacheName, project, userName + "v2");
}
public void clearSchemaCache(String project, String userName) {
- kylinCache.remove(Type.SCHEMA_CACHE.rootCacheName, project, userName);
+ removeCache(Type.SCHEMA_CACHE.rootCacheName, project, userName);
}
public void onClearSchemaCache(String project) {
@@ -314,12 +320,12 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
}
public void clearSchemaCache(String project) {
- kylinCache.clearByType(Type.SCHEMA_CACHE.rootCacheName, project);
+ clearCacheByType(Type.SCHEMA_CACHE.rootCacheName, project);
}
public void clearQueryCache(SQLRequest request) {
- kylinCache.remove(Type.SUCCESS_QUERY_CACHE.rootCacheName, request.getProject(), request.getCacheKey());
- kylinCache.remove(Type.EXCEPTION_QUERY_CACHE.rootCacheName, request.getProject(), request.getCacheKey());
+ removeCache(Type.SUCCESS_QUERY_CACHE.rootCacheName, request.getProject(), request.getCacheKey());
+ removeCache(Type.EXCEPTION_QUERY_CACHE.rootCacheName, request.getProject(), request.getCacheKey());
}
public void onClearProjectCache(String project) {
@@ -329,20 +335,44 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
public void clearProjectCache(String project) {
if (project == null) {
logger.info("[query cache log] clear query cache for all projects.");
- kylinCache.clearAll();
+ clearAllCache();
} else {
logger.info("[query cache log] clear query cache for {}", project);
- kylinCache.clearByType(Type.SUCCESS_QUERY_CACHE.rootCacheName, project);
- kylinCache.clearByType(Type.EXCEPTION_QUERY_CACHE.rootCacheName, project);
- kylinCache.clearByType(Type.SCHEMA_CACHE.rootCacheName, project);
+ clearCacheByType(Type.SUCCESS_QUERY_CACHE.rootCacheName, project);
+ clearCacheByType(Type.EXCEPTION_QUERY_CACHE.rootCacheName, project);
+ clearCacheByType(Type.SCHEMA_CACHE.rootCacheName, project);
}
}
- public void recoverCache() {
- boolean isRedisEnabled = KylinConfig.getInstanceFromEnv().isRedisEnabled();
- if (isRedisEnabled) {
- RedisCache.recoverInstance();
- logger.info("[query cache log] Redis client recover successfully.");
+ public boolean recoverCache() {
+ logger.info("Redis client recovery start.");
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ // no need to recover
+ if (!kylinConfig.isRedisEnabled()) {
+ logger.info("No need to recover Redis client, isRedisEnabled:{}", kylinConfig.isRedisEnabled());
+ return false;
+ }
+
+ if (!kylinConfig.isRedisSentinelEnabled()) {
+ // cluster、standalone
+ kylinCache = RedisCache.recoverInstance();
+ } else {
+ // sentinel
+ if (kylinCache instanceof RedisCacheV2) {
+ kylinCache = ((RedisCacheV2) kylinCache).recoverInstance();
+ } else {
+ kylinCache = RedisCacheV2.getInstance();
+ }
+ }
+
+ // recover failed，use Ehcache
+ if (kylinCache == null) {
+ logger.info("Redis client recover failed, use ehcache instead.");
+ kylinCache = KylinEhCache.getInstance();
+ return false;
+ } else {
+ logger.info("Redis client recover successfully.");
+ return true;
}
}
@@ -350,4 +380,47 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
public KylinCache getCache() {
return kylinCache;
}
+
+ public Object getCache(String type, String project, Object key) {
+ try {
+ return kylinCache.get(type, project, key);
+ } catch (Exception e) {
+ logger.error("Get cache failed, type:{}, project:{}, key:{}, exception:{}", type, project, key, e);
+ }
+ return null;
+ }
+
+ public void putCache(String type, String project, Object key, Object value) {
+ try {
+ kylinCache.put(type, project, key, value);
+ } catch (Exception e) {
+ logger.error("Put cache failed, type:{}, project:{}, key:{}, value:{}, exception:{}", type, project, key,
+ value, e);
+ }
+ }
+
+ public boolean removeCache(String type, String project, Object key) {
+ try {
+ return kylinCache.remove(type, project, key);
+ } catch (Exception e) {
+ logger.error("Remove cache failed, type:{}, project:{}, key:{}, exception:{}", type, project, key, e);
+ }
+ return false;
+ }
+
+ public void clearAllCache() {
+ try {
+ kylinCache.clearAll();
+ } catch (Exception e) {
+ logger.error("Clear all cache failed, exception:", e);
+ }
+ }
+
+ public void clearCacheByType(String type, String project) {
+ try {
+ kylinCache.clearByType(type, project);
+ } catch (Exception e) {
+ logger.error("Clear cache by type failed, type:{}, project:{}, exception:{}", type, project, e);
+ }
+ }
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryCacheManagerTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryCacheManagerTest.java
new file mode 100644
index 0000000000..5067356099
--- /dev/null
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryCacheManagerTest.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.kylin.common.SystemPropertiesCache;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.rest.cache.KylinCache;
+import org.apache.kylin.rest.cache.RedisCache;
+import org.apache.kylin.rest.cache.RedisCacheV2;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.test.util.ReflectionTestUtils;
+
+@MetadataInfo
+@ExtendWith(MockitoExtension.class)
+class QueryCacheManagerTest {
+ @Mock
+ private KylinCache kylinCache;
+ @InjectMocks
+ private QueryCacheManager queryCacheManager;
+
+ @Test
+ void testInitIsRedisEnabled() {
+ System.setProperty("kylin.cache.redis.enabled", "true");
+ try {
+ queryCacheManager.init();
+ } catch (Exception e) {
+ Assertions.fail();
+ }
+ System.clearProperty("kylin.cache.redis.enabled");
+ }
+
+ @Test
+ void testInitIsNotRedisEnabled() {
+ System.setProperty("kylin.cache.redis.enabled", "false");
+ try {
+ queryCacheManager.init();
+ } catch (Exception e) {
+ Assertions.fail();
+ }
+ System.clearProperty("kylin.cache.redis.enabled");
+ }
+
+ @Test
+ void testInitIsRedisSentinelEnabled() {
+ System.setProperty("kylin.cache.redis.enabled", "true");
+ System.setProperty("kylin.cache.redis.sentinel-enabled", "true");
+ System.setProperty("kylin.cache.redis.sentinel-master", "master");
+
+ assertDoesNotThrow(() -> queryCacheManager.init());
+
+ System.clearProperty("kylin.cache.redis.enabled");
+ System.clearProperty("kylin.cache.redis.sentinel-enabled");
+ System.clearProperty("kylin.cache.redis.sentinel-master");
+ }
+
+ @Test
+ void testRecoverCacheWhenNoNeedToRecover() {
+ System.setProperty("kylin.cache.redis.enabled", "false");
+
+ boolean recoverResult = queryCacheManager.recoverCache();
+ assertFalse(recoverResult);
+
+ System.clearProperty("kylin.cache.redis.enabled");
+ }
+
+ @Test
+ void testRecoverCacheWhenCluster() {
+ System.setProperty("kylin.cache.redis.enabled", "true");
+ kylinCache = Mockito.mock(RedisCache.class);
+ ReflectionTestUtils.setField(queryCacheManager, "kylinCache", kylinCache);
+
+ try {
+ queryCacheManager.recoverCache();
+ } catch (Exception e) {
+ Assertions.fail();
+ }
+
+ System.clearProperty("kylin.cache.redis.enabled");
+ }
+
+ @Test
+ void testRecoverCacheWhenSentinelRecoverTrue() {
+ SystemPropertiesCache.setProperty("kylin.cache.redis.enabled", "true");
+ SystemPropertiesCache.setProperty("kylin.cache.redis.sentinel-enabled", "true");
+ kylinCache = Mockito.mock(RedisCacheV2.class);
+ ReflectionTestUtils.setField(queryCacheManager, "kylinCache", kylinCache);
+ Mockito.when(((RedisCacheV2) kylinCache).recoverInstance()).thenReturn(kylinCache);
+
+ boolean recoverResult = queryCacheManager.recoverCache();
+ assertTrue(recoverResult);
+
+ SystemPropertiesCache.clearProperty("kylin.cache.redis.enabled");
+ SystemPropertiesCache.clearProperty("kylin.cache.redis.sentinel-enabled");
+ }
+
+ @Test
+ void testRecoverCacheWhenSentinelRecoverFailed() {
+ System.setProperty("kylin.cache.redis.enabled", "true");
+ System.setProperty("kylin.cache.redis.sentinel-enabled", "true");
+ kylinCache = Mockito.mock(RedisCacheV2.class);
+ ReflectionTestUtils.setField(queryCacheManager, "kylinCache", kylinCache);
+
+ boolean recoverResult = queryCacheManager.recoverCache();
+ assertFalse(recoverResult);
+
+ System.clearProperty("kylin.cache.redis.enabled");
+ System.clearProperty("kylin.cache.redis.sentinel-enabled");
+ }
+
+ @Test
+ void testGetCacheException() {
+ Mockito.when(kylinCache.get("type", "project", "key")).thenThrow(new RuntimeException());
+
+ Object value = queryCacheManager.getCache("type", "project", "key");
+
+ assertNull(value);
+ }
+
+ @Test
+ void testPutCacheException() {
+ Mockito.doThrow(new RuntimeException()).when(kylinCache).put("type", "project", "key", "value");
+
+ assertDoesNotThrow(() -> queryCacheManager.putCache("type", "project", "key", "value"));
+ }
+
+ @Test
+ void testRemoveCacheException() {
+ Mockito.doThrow(new RuntimeException()).when(kylinCache).remove("type", "project", "key");
+
+ boolean value = queryCacheManager.removeCache("type", "project", "key");
+
+ assertFalse(value);
+ }
+
+ @Test
+ void testClearAllCacheException() {
+ Mockito.doThrow(new RuntimeException()).when(kylinCache).clearAll();
+
+ assertDoesNotThrow(() -> queryCacheManager.clearAllCache());
+ }
+
+ @Test
+ void testClearCacheByTypeException() {
+ Mockito.doThrow(new RuntimeException()).when(kylinCache).clearByType("type", "project");
+
+ assertDoesNotThrow(() -> queryCacheManager.clearCacheByType("type", "project"));
+ }
+
+ @Test
+ void testClearSchemaCacheV2() {
+ assertDoesNotThrow(() -> queryCacheManager.clearSchemaCacheV2("project", "userName"));
+ }
+
+ @Test
+ void testClearSchemaCache() {
+ assertDoesNotThrow(() -> queryCacheManager.clearSchemaCache("project", "userName"));
+ }
+
+ @Test
+ void testOnClearSchemaCache() {
+ assertDoesNotThrow(() -> queryCacheManager.onClearProjectCache("project"));
+ }
+
+}
(kylin) 20/35: KYLIN-5816 Add zk lock for dict v3
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2159699d8e54588d879e0ebdbb1f2ed11380bb23
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Thu Sep 21 16:51:14 2023 +0800
KYLIN-5816 Add zk lock for dict v3
---
.../engine/spark/builder/DFDictionaryBuilder.scala | 17 +-----
.../kylin/engine/spark/builder/ZKHelper.scala | 42 +++++++++++++
.../spark/builder/v3dict/DictionaryBuilder.scala | 69 ++++++++++------------
3 files changed, 75 insertions(+), 53 deletions(-)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
index ae95344cd0..af8399ca3a 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
@@ -48,28 +48,13 @@ class DFDictionaryBuilder(
changeAQEConfig(true)
}
- private val YARN_CLUSTER: String = "cluster"
private val AQE = "spark.sql.adaptive.enabled";
private val originalAQE = ss.conf.get(AQE)
- private def tryZKJaasConfiguration(): Unit = {
- val config = KylinConfig.getInstanceFromEnv
- if (YARN_CLUSTER.equals(config.getDeployMode)) {
- val kapConfig = KapConfig.wrap(config)
- if (KapConfig.FI_PLATFORM.equals(kapConfig.getKerberosPlatform) || KapConfig.TDH_PLATFORM.equals(kapConfig.getKerberosPlatform)) {
- val sparkConf = ss.sparkContext.getConf
- val principal = sparkConf.get("spark.kerberos.principal")
- val keytab = sparkConf.get("spark.kerberos.keytab")
- logInfo(s"ZKJaasConfiguration principal: $principal, keyTab: $keytab")
- javax.security.auth.login.Configuration.setConfiguration(new ZKJaasConfiguration(principal, keytab))
- }
- }
- }
-
@throws[IOException]
private[builder] def safeBuild(ref: TblColRef): Unit = {
val sourceColumn = ref.getIdentity
- tryZKJaasConfiguration()
+ ZKHelper.tryZKJaasConfiguration(ss)
val lock: Lock = KylinConfig.getInstanceFromEnv.getDistributedLockFactory
.getLockForCurrentThread(getLockPath(sourceColumn))
lock.lock()
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala
new file mode 100644
index 0000000000..1fd321f64e
--- /dev/null
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.spark.builder
+
+import org.apache.kylin.common.{KapConfig, KylinConfig}
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+
+object ZKHelper extends Logging{
+ private val YARN_CLUSTER: String = "cluster"
+
+ def tryZKJaasConfiguration(ss: SparkSession): Unit = {
+ val config = KylinConfig.getInstanceFromEnv
+ if (YARN_CLUSTER.equals(config.getDeployMode)) {
+ val kapConfig = KapConfig.wrap(config)
+ if (KapConfig.FI_PLATFORM.equals(kapConfig.getKerberosPlatform) || KapConfig.TDH_PLATFORM.equals(kapConfig.getKerberosPlatform)) {
+ val sparkConf = ss.sparkContext.getConf
+ val principal = sparkConf.get("spark.kerberos.principal")
+ val keytab = sparkConf.get("spark.kerberos.keytab")
+ logInfo(s"ZKJaasConfiguration principal: $principal, keyTab: $keytab")
+ javax.security.auth.login.Configuration.setConfiguration(new ZKJaasConfiguration(principal, keytab))
+ }
+ }
+ }
+
+}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
index 5fd9ae1673..9938b1e00c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.exception.KylinRuntimeException
import org.apache.kylin.common.util.HadoopUtil
+import org.apache.kylin.engine.spark.builder.ZKHelper
import org.apache.kylin.engine.spark.builder.v3dict.DictBuildMode.{V2UPGRADE, V3APPEND, V3INIT, V3UPGRADE}
import org.apache.kylin.engine.spark.job.NSparkCubingUtil
import org.apache.kylin.metadata.model.TblColRef
@@ -40,7 +41,7 @@ import util.retry.blocking.RetryStrategy.RetryStrategyProducer
import util.retry.blocking.{Failure, Retry, RetryStrategy, Success}
import java.nio.file.Paths
-import java.util.concurrent.locks.ReentrantReadWriteLock
+import java.util.concurrent.locks.Lock
import scala.collection.mutable.ListBuffer
import scala.concurrent.duration.DurationInt
import scala.util.control.NonFatal
@@ -49,7 +50,6 @@ object DictionaryBuilder extends Logging {
implicit val retryStrategy: RetryStrategyProducer =
RetryStrategy.randomBackOff(5.seconds, 15.seconds, maxAttempts = 20)
- lazy val v3dictMergeLock = new ReentrantReadWriteLock(true)
def buildGlobalDict(
project: String,
@@ -88,12 +88,11 @@ object DictionaryBuilder extends Logging {
private def transformerDictPlan(
spark: SparkSession,
context: DictionaryContext,
- plan: LogicalPlan): DictPlanVersion = {
+ plan: LogicalPlan): LogicalPlan = {
val dictPath = getDictionaryPathAndCheck(context)
- val dictVersion = DeltaLog.forTable(spark, dictPath).snapshot.version
- val dictTableDF = spark.read.format("delta").option("versionAsOf", dictVersion).load(dictPath);
- val maxOffset = dictTableDF.count()
+ val dictTable: DeltaTable = DeltaTable.forPath(dictPath)
+ val maxOffset = dictTable.toDF.count()
logInfo(s"Dict $dictPath item count $maxOffset")
plan match {
@@ -102,19 +101,19 @@ object DictionaryBuilder extends Logging {
val windowSpec = org.apache.spark.sql.expressions.Window.orderBy(col(column))
val joinCondition = createColumn(
EqualTo(col(column).cast(StringType).expr,
- getLogicalPlan(dictTableDF).output.head))
- val filterKey = getLogicalPlan(dictTableDF).output.head.name
+ getLogicalPlan(dictTable.toDF).output.head))
+ val filterKey = getLogicalPlan(dictTable.toDF).output.head.name
val antiJoinDF = getDataFrame(spark, windowChild)
.filter(col(filterKey).isNotNull)
- .join(dictTableDF,
+ .join(dictTable.toDF,
joinCondition,
"left_anti")
.select(col(column).cast(StringType) as "dict_key",
(row_number().over(windowSpec) + lit(maxOffset)).cast(LongType) as "dict_value")
logInfo(s"Dict logical plan : ${antiJoinDF.queryExecution.logical.treeString}")
- DictPlanVersion(getLogicalPlan(antiJoinDF), dictVersion)
+ getLogicalPlan(antiJoinDF)
- case _ => DictPlanVersion(plan, dictVersion)
+ case _ => plan
}
}
@@ -186,34 +185,22 @@ object DictionaryBuilder extends Logging {
}
private def mergeIncrementDict(spark: SparkSession, context: DictionaryContext, plan: LogicalPlan): Unit = {
- val incDictVersion = transformerDictPlan(spark, context, plan)
- val incrementDictDF = getDataFrame(spark, incDictVersion.incDictPlan)
- val dictPath = getDictionaryPathAndCheck(context)
- if (incrementDictDF.isEmpty) {
- logInfo(s"Increment dict for global dict $dictPath is empty, no need to merge.")
- return
- }
- tryMergeIncrementDict(spark, dictPath, incDictVersion.sourceDeltaVersion, incrementDictDF)
- }
-
- private def tryMergeIncrementDict(spark: SparkSession, dictPath: String, dictVersion: Long,
- incDictDF: Dataset[Row]): Unit = {
- v3dictMergeLock.writeLock().lock()
+ ZKHelper.tryZKJaasConfiguration(spark)
+ val lock: Lock = KylinConfig.getInstanceFromEnv.getDistributedLockFactory
+ .getLockForCurrentThread(getDictionaryLockPath(context))
+ lock.lock()
try {
+ val dictPlan = transformerDictPlan(spark, context, plan)
+ val incrementDictDF = getDataFrame(spark, dictPlan)
+ val dictPath = getDictionaryPathAndCheck(context)
logInfo(s"Increment build global dict $dictPath")
- val curVersion = DeltaLog.forTable(spark, dictPath).snapshot.version
- if (dictVersion != curVersion) {
- logInfo(s"Cur v3dict version is $curVersion, incDict is based on version $curVersion, will be retry")
- throw new KylinRuntimeException(s"Cur v3dict version is $curVersion, " +
- s"incDict is based on version $curVersion, will be retry")
- } else {
- val dictTable = DeltaTable.forPath(dictPath)
- dictTable.merge(incDictDF, "1 != 1")
- .whenNotMatched().insertAll()
- .execute()
- }
+ val dictTable = DeltaTable.forPath(dictPath)
+ dictTable.alias("dict")
+ .merge(incrementDictDF.alias("incre_dict"), "1 != 1")
+ .whenNotMatched().insertAll()
+ .execute()
} finally {
- v3dictMergeLock.writeLock().unlock()
+ lock.unlock()
}
}
@@ -353,11 +340,19 @@ object DictionaryBuilder extends Logging {
v3ditPath
}
+ def getDictionaryLockPath(context: DictionaryContext): String = {
+ val dictPath = Paths.get(context.project,
+ HadoopUtil.GLOBAL_DICT_V3_STORAGE_ROOT,
+ context.dbName,
+ context.tableName,
+ context.columnName)
+ dictPath.toString
+ }
+
def wrapCol(ref: TblColRef): String = {
NSparkCubingUtil.convertFromDot(ref.getBackTickIdentity)
}
}
-case class DictPlanVersion(incDictPlan: LogicalPlan, sourceDeltaVersion: Long)
class DictionaryContext(
val project: String,
(kylin) 13/35: KYLIN-5807 Fix query execute as user without data query acl
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a212350a518cde5fa1fe0357a71b806f5533e2c3
Author: Zhimin Wu <59...@qq.com>
AuthorDate: Fri Sep 1 18:26:22 2023 +0800
KYLIN-5807 Fix query execute as user without data query acl
Co-authored-by: Feng Zhu <fi...@outlook.com>
---
.../java/org/apache/kylin/rest/service/AccessService.java | 14 ++++++++++----
.../java/org/apache/kylin/rest/service/QueryService.java | 13 +++++++++++++
.../org/apache/kylin/rest/service/QueryServiceTest.java | 2 ++
3 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 54dcf242c8..031282378f 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -756,13 +756,19 @@ public class AccessService extends BasicService {
return sidWithPermissions;
}
- @SneakyThrows(IOException.class)
public Set<String> getUserNormalExtPermissions(String project) {
- String projectUuid = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
- .getUuid();
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
if (Objects.nonNull(authentication)) {
- val userName = authentication.getName();
+ return getUserNormalExtPermissionsByUserInProject(project, authentication.getName());
+ }
+ return new HashSet<>();
+ }
+
+ @SneakyThrows(IOException.class)
+ public Set<String> getUserNormalExtPermissionsByUserInProject(String project, String userName) {
+ String projectUuid = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
+ .getUuid();
+ if (Objects.nonNull(userName)) {
if (userAclService.canAdminUserQuery(userName)) {
return Collections.singleton(AclConstants.DATA_QUERY);
}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 60837454a0..7b10e1eb67 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -81,6 +81,7 @@ import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.constants.AclConstants;
import org.apache.kylin.engine.spark.filter.BloomFilterSkipCollector;
import org.apache.kylin.engine.spark.filter.ParquetPageFilterCollector;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
@@ -567,6 +568,18 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
if (!grantedProjects.contains(sqlRequest.getProject())) {
throw new KylinException(ACCESS_DENIED, "Access is denied.");
}
+
+ val isDataPermissionDefaultEnabled = KylinConfig.getInstanceFromEnv().isDataPermissionDefaultEnabled();
+ if (isDataPermissionDefaultEnabled) {
+ try {
+ Set<String> extPermissions = accessService.getUserNormalExtPermissionsByUserInProject(sqlRequest.getProject(), executeUser);
+ if (!extPermissions.contains(AclConstants.DATA_QUERY)) {
+ throw new KylinException(ACCESS_DENIED, "Access is denied.");
+ }
+ } catch (Exception e) {
+ throw new KylinException(ACCESS_DENIED, e);
+ }
+ }
}
private void checkSqlRequest(SQLRequest sqlRequest) {
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 200fbb960f..89fead0d99 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -230,6 +230,8 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
ReflectionTestUtils.setField(queryService, "aclTCRService", aclTCRService);
ReflectionTestUtils.setField(accessService, "userService", userService);
ReflectionTestUtils.setField(accessService, "aclService", aclService);
+ ReflectionTestUtils.setField(accessService, "userAclService", userAclService);
+ ReflectionTestUtils.setField(userAclService, "userService", userService);
ReflectionTestUtils.setField(aclTCRService, "accessService", accessService);
ReflectionTestUtils.setField(aclTCRService, "userService", userService);
ReflectionTestUtils.setField(queryService, "appConfig", appConfig);
(kylin) 16/35: KYLIN-5810 Add the spark.session.isDefined check in queryExec
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit dcfa26067ff19352e064801d191f27c98fe1e6d2
Author: Yinghao Lin <39...@users.noreply.github.com>
AuthorDate: Wed Sep 6 14:04:06 2023 +0800
KYLIN-5810 Add the spark.session.isDefined check in queryExec
---
.../src/main/java/org/apache/kylin/query/engine/QueryExec.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index d9015fb340..442eea7faf 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -78,6 +78,7 @@ import org.apache.kylin.query.util.QueryContextCutter;
import org.apache.kylin.query.util.QueryInterruptChecker;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.RelAggPushDownUtil;
+import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,7 +274,7 @@ public class QueryExec {
postOptRules.addAll(HepUtils.AggPushDownRules);
}
- if(kylinConfig.isScalarSubqueryJoinEnabled()) {
+ if (kylinConfig.isScalarSubqueryJoinEnabled()) {
postOptRules.addAll(HepUtils.ScalarSubqueryJoinRules);
}
@@ -598,14 +599,16 @@ public class QueryExec {
}
private String processAcceptCacheTimeInSql(String sql) {
- if (kylinConfig.isKylinLocalCacheEnabled() && kylinConfig.isKylinFileStatusCacheEnabled()) {
+ if (kylinConfig.isKylinLocalCacheEnabled() && kylinConfig.isKylinFileStatusCacheEnabled()
+ && SparkSession.getDefaultSession().isDefined()) {
return KylinCacheFileSystem.processAcceptCacheTimeInSql(sql);
}
return sql;
}
private void clearAcceptCacheTimeLocally() {
- if (kylinConfig.isKylinLocalCacheEnabled() && kylinConfig.isKylinFileStatusCacheEnabled()) {
+ if (kylinConfig.isKylinLocalCacheEnabled() && kylinConfig.isKylinFileStatusCacheEnabled()
+ && SparkSession.getDefaultSession().isDefined()) {
KylinCacheFileSystem.clearAcceptCacheTimeLocally();
}
}
(kylin) 33/35: KYLIN-5844 & KYLIN-5843 Column ACL works for PushDown
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e804eee81eff19424ea61c6899c226ba3d5fcf59
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Sun Oct 1 17:19:17 2023 +0800
KYLIN-5844 & KYLIN-5843 Column ACL works for PushDown
1. Column ACL works for PushDown
2. Flat-table sql should consider acl
---
.../apache/kylin/metadata/model/ColumnDesc.java | 5 +-
.../apache/kylin/rest/util/AclPermissionUtil.java | 2 +-
.../kylin/rest/response/ExecutableResponse.java | 4 +-
.../routing/DataflowCapabilityCheckerTest.java | 3 +-
.../kylin/query/rules/AggPushdownRuleTest.java | 4 +-
.../kylin/query/rules/CalciteRuleTestBase.java | 12 +-
.../rest/response/LayoutRecDetailResponse.java | 2 +-
.../apache/kylin/rest/service/ModelService.java | 10 +-
.../rest/service/params/BasicSegmentParams.java | 3 +-
.../security/HackSelectStarWithColumnACL.java | 259 +++++++------
.../org/apache/kylin/query/util/PushDownUtil.java | 32 +-
.../security/HackSelectStarWithColumnACLTest.java | 405 ++++++++++++++-------
.../apache/kylin/query/util/PushDownUtilTest.java | 359 ++++++++++--------
.../merger/AfterMergeOrRefreshResourceMerger.java | 3 +-
.../spark/source/NSparkMetadataExplorer.java | 3 +-
.../engine/spark/job/NSparkMergingJobTest.java | 10 +-
16 files changed, 705 insertions(+), 411 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index c2efaccd4e..1754104ae4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -23,6 +23,7 @@ import java.io.Serializable;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -30,7 +31,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.Getter;
import lombok.Setter;
@@ -231,8 +231,7 @@ public class ColumnDesc implements Serializable {
// for test mainly
public static ColumnDesc mockup(TableDesc table, int oneBasedColumnIndex, String name, String datatype) {
ColumnDesc desc = new ColumnDesc();
- String id = "" + oneBasedColumnIndex;
- desc.setId(id);
+ desc.setId(String.valueOf(oneBasedColumnIndex));
desc.setName(name);
desc.setDatatype(datatype);
desc.init(table);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
index 36360a0f12..82aaca58dc 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
@@ -235,7 +235,7 @@ public class AclPermissionUtil {
}
public static QueryContext.AclInfo createAclInfo(String project, Set<String> groups) {
- return new QueryContext.AclInfo(getCurrentUsername(), groups, isAdminInProject(project, groups));
+ return new QueryContext.AclInfo(getCurrentUsername(), groups, hasProjectAdminPermission(project, groups));
}
public static boolean hasExtPermission(Permission permission) {
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
index 70144b83a2..6736c2bb74 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
@@ -27,6 +27,8 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
import org.apache.kylin.engine.spark.job.NTableSamplingJob;
import org.apache.kylin.job.SecondStorageCleanJobUtil;
@@ -43,11 +45,9 @@ import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
import org.apache.kylin.metadata.model.TableDesc;
-import com.clearspring.analytics.util.Lists;
import com.fasterxml.jackson.annotation.JsonManagedReference;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonUnwrapped;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.AllArgsConstructor;
import lombok.Getter;
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
index 561fa50380..1865372e8e 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/DataflowCapabilityCheckerTest.java
@@ -27,6 +27,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
@@ -51,8 +52,6 @@ import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.Assert;
import org.junit.Test;
-import com.clearspring.analytics.util.Lists;
-
public class DataflowCapabilityCheckerTest extends NLocalWithSparkSessionTest {
private final long baseLayoutId = IndexEntity.TABLE_INDEX_START_ID + 1;
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/AggPushdownRuleTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/AggPushdownRuleTest.java
index dfe7f9fca3..eec9bb41bf 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/AggPushdownRuleTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/AggPushdownRuleTest.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.kylin.query.rules;
import java.io.IOException;
@@ -28,12 +27,11 @@ import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.test.DiffRepository;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import com.clearspring.analytics.util.Lists;
-
import io.kyligence.kap.query.optrule.KapAggFilterTransposeRule;
import io.kyligence.kap.query.optrule.KapAggJoinTransposeRule;
import io.kyligence.kap.query.optrule.KapAggProjectMergeRule;
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java
index 01a96c118f..f152f903b4 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java
@@ -20,7 +20,6 @@ package org.apache.kylin.query.rules;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.junit.Assert.assertThat;
import java.io.IOException;
import java.util.Collection;
@@ -42,6 +41,8 @@ import org.apache.commons.lang3.NotImplementedException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.base.Strings;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.QueryOptimizer;
import org.apache.kylin.query.util.HepUtils;
@@ -53,9 +54,6 @@ import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.clearspring.analytics.util.Lists;
-import org.apache.kylin.guava30.shaded.common.base.Strings;
-
public class CalciteRuleTestBase extends NLocalFileMetadataTestCase {
private static final Logger logger = LoggerFactory.getLogger(CalciteRuleTestBase.class);
@@ -183,13 +181,13 @@ public class CalciteRuleTestBase extends NLocalFileMetadataTestCase {
}
protected void checkPlanning(RelNode relBefore, RelNode relAfter, String prefix, boolean unchanged) {
- assertThat(relBefore, notNullValue());
- assertThat(relAfter, notNullValue());
+ Assert.assertThat(relBefore, notNullValue());
+ Assert.assertThat(relAfter, notNullValue());
final String planBefore = NL + RelOptUtil.toString(relBefore);
final String planAfter = NL + RelOptUtil.toString(relAfter);
if (unchanged) {
- assertThat(planAfter, is(planBefore));
+ Assert.assertThat(planAfter, is(planBefore));
} else {
checkDiff(relBefore, relAfter, prefix);
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
index 7b8959e374..bfcbfc964b 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
@@ -20,10 +20,10 @@ package org.apache.kylin.rest.response;
import java.io.Serializable;
import java.util.List;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
-import com.clearspring.analytics.util.Lists;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 5e2143a39a..a3b1773ff8 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -1311,8 +1311,11 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public String getModelSql(String modelId, String project) {
aclEvaluate.checkProjectReadPermission(project);
- NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
- return PushDownUtil.generateFlatTableSql(model, false);
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.setAclInfo(AclPermissionUtil.createAclInfo(project, getCurrentUserGroups()));
+ NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
+ return PushDownUtil.generateFlatTableSql(model, false);
+ }
}
public List<RelatedModelResponse> getRelateModels(String project, String table, String modelId) {
@@ -1751,9 +1754,10 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return;
}
- try {
+ try (QueryContext queryContext = QueryContext.current()) {
String project = model.getProject();
ProjectInstance prjInstance = getManager(NProjectManager.class).getProject(project);
+ queryContext.setAclInfo(AclPermissionUtil.createAclInfo(project, getCurrentUserGroups()));
if (prjInstance.getSourceType() == ISourceAware.ID_SPARK
&& model.getModelType() == NDataModel.ModelType.BATCH) {
SparkSession ss = SparderEnv.getSparkSession();
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/BasicSegmentParams.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/BasicSegmentParams.java
index 5190456985..ee4eef9f21 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/BasicSegmentParams.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/BasicSegmentParams.java
@@ -20,11 +20,10 @@ package org.apache.kylin.rest.service.params;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.rest.aspect.TransactionProjectUnit;
-import com.clearspring.analytics.util.Lists;
-
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java b/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java
index fb4996ff17..a6112e47fa 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java
@@ -18,118 +18,51 @@
package org.apache.kylin.query.security;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
-import org.apache.calcite.avatica.util.Quoting;
+import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlExplain;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlJoin;
+import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlWith;
+import org.apache.calcite.sql.SqlWithItem;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.util.SqlBasicVisitor;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinRuntimeException;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.metadata.acl.AclTCR;
import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.IQueryTransformer;
-import org.apache.kylin.query.exception.NoAuthorizedColsError;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import lombok.Getter;
public class HackSelectStarWithColumnACL implements IQueryTransformer, IPushDownConverter {
- private static final String SELECT_STAR = "*";
-
- static String getNewSelectClause(SqlNode sqlNode, String project, String defaultSchema,
- QueryContext.AclInfo aclInfo) {
- StringBuilder newSelectClause = new StringBuilder();
- List<String> allCols = getColsCanAccess(sqlNode, project, defaultSchema, aclInfo);
- if (CollectionUtils.isEmpty(allCols)) {
- throw new NoAuthorizedColsError();
- }
- for (String col : allCols) {
- if (!col.equals(allCols.get(allCols.size() - 1))) {
- newSelectClause.append(col).append(", ");
- } else {
- newSelectClause.append(col);
- }
- }
- return newSelectClause.toString();
- }
-
- static List<String> getColsCanAccess(SqlNode sqlNode, String project, String defaultSchema,
- QueryContext.AclInfo aclInfo) {
- List<String> cols = new ArrayList<>();
- String user = Objects.nonNull(aclInfo) ? aclInfo.getUsername() : null;
- Set<String> groups = Objects.nonNull(aclInfo) ? aclInfo.getGroups() : null;
- final List<AclTCR> aclTCRs = AclTCRManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
- .getAclTCRs(user, groups);
- List<RowFilter.Table> tblWithAlias = RowFilter.getTblWithAlias(defaultSchema, getSingleSelect(sqlNode));
- for (RowFilter.Table table : tblWithAlias) {
- TableDesc tableDesc = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
- .getTableDesc(table.getName());
- if (Objects.isNull(tableDesc)) {
- throw new IllegalStateException(
- "Table " + table.getAlias() + " not found. Please add table " + table.getAlias()
- + " to data source. If this table does exist, mention it as DATABASE.TABLE.");
- }
-
- List<ColumnDesc> columns = Lists.newArrayList(tableDesc.getColumns());
- Collections.sort(columns, Comparator.comparing(ColumnDesc::getZeroBasedIndex));
- String quotingChar = Quoting.valueOf(KylinConfig.getInstanceFromEnv().getCalciteQuoting()).string;
- for (ColumnDesc column : columns) {
- if (aclTCRs.stream()
- .anyMatch(aclTCR -> aclTCR.isAuthorized(tableDesc.getIdentity(), column.getName()))) {
- StringBuilder sb = new StringBuilder();
- sb.append(quotingChar).append(table.getAlias()).append(quotingChar) //
- .append('.') //
- .append(quotingChar).append(column.getName()).append(quotingChar);
- cols.add(sb.toString());
- }
- }
- }
- return cols;
- }
-
- private static boolean isSingleSelectStar(SqlNode sqlNode) {
- if (SelectNumVisitor.getSelectNum(sqlNode) != 1 || sqlNode instanceof SqlExplain) {
- return false;
- }
- SqlSelect singleSelect = getSingleSelect(sqlNode);
- return singleSelect.getSelectList().toString().equals(SELECT_STAR);
- }
-
- private static int getSelectStarPos(String sql, SqlNode sqlNode) {
- SqlSelect singleSelect = getSingleSelect(sqlNode);
- Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(singleSelect.getSelectList(), sql);
- Preconditions.checkState(replacePos.getSecond() - replacePos.getFirst() == 1);
- return replacePos.getFirst();
- }
-
- private static SqlSelect getSingleSelect(SqlNode sqlNode) {
- if (sqlNode instanceof SqlOrderBy) {
- SqlOrderBy orderBy = (SqlOrderBy) sqlNode;
- return (SqlSelect) orderBy.query;
- } else {
- return (SqlSelect) sqlNode;
- }
- }
-
private static boolean hasAdminPermission(QueryContext.AclInfo aclInfo) {
if (Objects.isNull(aclInfo) || Objects.isNull(aclInfo.getGroups())) {
return false;
@@ -153,49 +86,161 @@ public class HackSelectStarWithColumnACL implements IQueryTransformer, IPushDown
try {
sqlNode = CalciteParser.parse(sql, project);
} catch (SqlParseException e) {
- throw new KylinRuntimeException("Failed to parse SQL \'" + sql + "\', please make sure the SQL is valid");
+ throw new KylinRuntimeException("Failed to parse invalid SQL: " + sql);
}
- if (!isSingleSelectStar(sqlNode)) {
+ SelectStarAuthVisitor replacer = new SelectStarAuthVisitor(project, defaultSchema, aclLocal);
+ sqlNode.accept(replacer);
+ Map<SqlNode, String> resolved = replacer.getResolved();
+ if (resolved.isEmpty()) {
return sql;
}
-
- String newSelectClause = getNewSelectClause(sqlNode, project, defaultSchema, aclLocal);
- int selectStarPos = getSelectStarPos(sql, sqlNode);
- StringBuilder result = new StringBuilder(sql);
- result.replace(selectStarPos, selectStarPos + 1, newSelectClause);
- return result.toString();
+ StringBuilder sb = new StringBuilder();
+ AtomicInteger offset = new AtomicInteger(0);
+ resolved.forEach((node, replaced) -> {
+ Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(node, sql);
+ sb.append(sql, offset.get(), replacePos.getKey());
+ sb.append(replaced);
+ offset.set(replacePos.getValue());
+ });
+ sb.append(sql.substring(offset.get()));
+ return sb.toString();
}
- static class SelectNumVisitor extends SqlBasicVisitor<SqlNode> {
- int selectNum = 0;
+ static class SelectStarAuthVisitor extends SqlBasicVisitor<SqlNode> {
+ @Getter
+ Map<SqlNode, String> resolved = new LinkedHashMap<>();
+ /** A cache to avoid the same table generate the replaced sql twice. */
+ private final Map<String, String> tableToReplacedSubQuery = new HashMap<>();
+ private final Set<String> namesOfWithItems = new HashSet<>();
+ private final String defaultSchema;
+ private final List<AclTCR> aclTCRList;
+ private final NTableMetadataManager tableMgr;
+
+ SelectStarAuthVisitor(String project, String defaultSchema, QueryContext.AclInfo aclInfo) {
+ this.defaultSchema = defaultSchema;
+ KylinConfig config = NProjectManager.getProjectConfig(project);
+ this.tableMgr = NTableMetadataManager.getInstance(config, project);
+ // init aclTCR
+ String user = Objects.nonNull(aclInfo) ? aclInfo.getUsername() : null;
+ Set<String> groups = Objects.nonNull(aclInfo) ? aclInfo.getGroups() : null;
+ aclTCRList = AclTCRManager.getInstance(config, project).getAclTCRs(user, groups);
+ }
- static int getSelectNum(SqlNode sqlNode) {
- SelectNumVisitor snv = new SelectNumVisitor();
- sqlNode.accept(snv);
- return snv.getNum();
+ @Override
+ public SqlNode visit(SqlNodeList nodeList) {
+ for (SqlNode node : nodeList) {
+ if (node instanceof SqlWithItem) {
+ SqlWithItem item = (SqlWithItem) node;
+ item.query.accept(this);
+ namesOfWithItems.add(item.name.toString());
+ }
+ }
+ return null;
}
@Override
public SqlNode visit(SqlCall call) {
if (call instanceof SqlSelect) {
- selectNum++;
+ SqlSelect select = (SqlSelect) call;
+ markCall(select.getFrom());
+ }
+ if (call instanceof SqlBasicCall) {
+ if (isCallWithAlias(call)) {
+ markCall(call);
+ } else {
+ SqlBasicCall basicCall = (SqlBasicCall) call;
+ for (SqlNode node : basicCall.getOperands()) {
+ markCall(node);
+ }
+ }
+ }
+
+ if (call instanceof SqlJoin) {
+ markCall(((SqlJoin) call).getLeft());
+ markCall(((SqlJoin) call).getRight());
+ }
+ if (call instanceof SqlWith) {
+ SqlWith sqlWith = (SqlWith) call;
+ sqlWith.withList.accept(this);
+ sqlWith.body.accept(this);
}
if (call instanceof SqlOrderBy) {
- SqlOrderBy sqlOrderBy = (SqlOrderBy) call;
- sqlOrderBy.query.accept(this);
+ call.getOperandList().stream().filter(Objects::nonNull).forEach(node -> node.accept(this));
+ }
+ return null;
+ }
+
+ private void markCall(SqlNode operand) {
+ if (operand instanceof SqlIdentifier) {
+ String replaced = markTableIdentifier((SqlIdentifier) operand, null);
+ resolved.put(operand, replaced);
+ return;
+ } else if (isCallWithAlias(operand)) {
+ SqlNode[] operands = ((SqlBasicCall) operand).getOperands();
+ SqlNode tableNode = operands[0];
+ if (tableNode instanceof SqlIdentifier) {
+ String replaced = markTableIdentifier((SqlIdentifier) tableNode, operands[1]);
+ resolved.put(operand, replaced);
+ } else {
+ tableNode.accept(this);
+ }
+ return;
+ }
+
+ // a sub-query, continue to mark
+ operand.accept(this);
+ }
+
+ private boolean isCallWithAlias(SqlNode from) {
+ return from instanceof SqlBasicCall && from.getKind() == SqlKind.AS;
+ }
+
+ private String markTableIdentifier(SqlIdentifier operand, SqlNode alias) {
+ if (namesOfWithItems.contains(operand.toString())) {
+ return operand.toString();
+ }
+ List<String> names = operand.names;
+ String schema = names.size() == 1 ? defaultSchema : names.get(0);
+ String table = names.size() == 1 ? names.get(0) : names.get(1);
+ TableDesc tableDesc = tableMgr.getTableDesc(schema + '.' + table);
+ if (tableDesc == null) {
+ throw new KylinRuntimeException("Failed to parse table: " + operand);
+ }
+
+ String tableIdentity = tableDesc.getDoubleQuoteIdentity();
+ if (tableToReplacedSubQuery.containsKey(tableIdentity)) {
+ return tableToReplacedSubQuery.get(tableIdentity);
} else {
- for (SqlNode operand : call.getOperandList()) {
- if (operand != null) {
- operand.accept(this);
+ List<String> authorizedCols = getAuthorizedCols(tableDesc);
+ String subQueryAlias = alias == null ? StringHelper.doubleQuote(table)
+ : StringHelper.doubleQuote(alias.toString());
+ String replacedSubQuery = "( select " + String.join(", ", authorizedCols) //
+ + " from " + tableIdentity + ") as " + subQueryAlias;
+ tableToReplacedSubQuery.put(tableIdentity, replacedSubQuery);
+ return replacedSubQuery;
+ }
+ }
+
+ List<String> getAuthorizedCols(TableDesc tableDesc) {
+ List<String> colList = new ArrayList<>();
+ List<ColumnDesc> columns = Arrays.stream(tableDesc.getColumns()) //
+ .sorted(Comparator.comparing(ColumnDesc::getZeroBasedIndex)) //
+ .collect(Collectors.toList());
+ for (ColumnDesc column : columns) {
+ for (AclTCR aclTCR : aclTCRList) {
+ if (aclTCR.isAuthorized(tableDesc.getIdentity(), column.getName())) {
+ colList.add(getQuotedColName(column));
+ break;
}
}
}
- return null;
+ return colList;
}
- private int getNum() {
- return selectNum;
+ String getQuotedColName(ColumnDesc column) {
+ return StringHelper.doubleQuote(column.getTable().getName()) + "."
+ + StringHelper.doubleQuote(column.getName());
}
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index b2aeac6995..a4d79d7d8f 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.concurrent.ExecutionException;
@@ -58,6 +59,9 @@ import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.acl.AclTCR;
+import org.apache.kylin.metadata.acl.AclTCRManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.JoinDesc;
@@ -87,7 +91,7 @@ public class PushDownUtil {
private static final Logger logger = LoggerFactory.getLogger("query");
- // sql hint "/*+ MODEL_PRIORITY({cube_name}) */"
+ // sql hint "/*+ MODEL_PRIORITY({cube_name}) */"
private static final Pattern SQL_HINT_PATTERN = Pattern
.compile("/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/");
public static final String DEFAULT_SCHEMA = "DEFAULT";
@@ -244,7 +248,9 @@ public class PushDownUtil {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("SELECT ").append(sep);
- List<TblColRef> tblColRefs = Lists.newArrayList(model.getEffectiveCols().values());
+ List<TblColRef> originTblColRefs = Lists.newArrayList(model.getEffectiveCols().values());
+ List<TblColRef> tblColRefs = getAuthorizedCols(model.getProject(), originTblColRefs);
+
if (tblColRefs.isEmpty()) {
sqlBuilder.append("1 ").append(sep);
} else {
@@ -272,6 +278,28 @@ public class PushDownUtil {
return new EscapeTransformer().transform(sqlBuilder.toString());
}
+ private static List<TblColRef> getAuthorizedCols(String project, List<TblColRef> tableColRefs) {
+ QueryContext.AclInfo aclInfo = QueryContext.current().getAclInfo();
+ if (aclInfo != null && aclInfo.isHasAdminPermission()) {
+ return tableColRefs;
+ }
+ String user = Objects.nonNull(aclInfo) ? aclInfo.getUsername() : null;
+ Set<String> groups = Objects.nonNull(aclInfo) ? aclInfo.getGroups() : null;
+ KylinConfig config = NProjectManager.getProjectConfig(project);
+ List<AclTCR> aclTCRList = AclTCRManager.getInstance(config, project).getAclTCRs(user, groups);
+ List<TblColRef> result = Lists.newArrayList();
+ for (TblColRef tableColRef : tableColRefs) {
+ ColumnDesc column = tableColRef.getColumnDesc();
+ for (AclTCR aclTCR : aclTCRList) {
+ if (aclTCR.isAuthorized(tableColRef.getTableWithSchema(), column.getName())) {
+ result.add(tableColRef);
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
public static String expandComputedColumnExp(NDataModel model, String project, String expression) {
StringBuilder forCC = new StringBuilder();
forCC.append("select ").append(expression).append(" ,").append(CC_SPLITTER) //
diff --git a/src/query/src/test/java/org/apache/kylin/query/security/HackSelectStarWithColumnACLTest.java b/src/query/src/test/java/org/apache/kylin/query/security/HackSelectStarWithColumnACLTest.java
index 5fe5a9d491..4ed0a797d1 100644
--- a/src/query/src/test/java/org/apache/kylin/query/security/HackSelectStarWithColumnACLTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/security/HackSelectStarWithColumnACLTest.java
@@ -18,61 +18,281 @@
package org.apache.kylin.query.security;
-import java.util.ArrayList;
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+
import java.util.Arrays;
import java.util.List;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.QueryContext;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.metadata.acl.AclTCR;
import org.apache.kylin.metadata.acl.AclTCRManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.NTableMetadataManager;
-import org.apache.kylin.query.exception.NoAuthorizedColsError;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
-public class HackSelectStarWithColumnACLTest extends NLocalFileMetadataTestCase {
- private final static String PROJECT = "default";
- private final static String SCHEMA = "DEFAULT";
+@MetadataInfo
+class HackSelectStarWithColumnACLTest {
+ private static final String PROJECT = "default";
+ private static final String SCHEMA = "DEFAULT";
+ private static final HackSelectStarWithColumnACL TRANSFORMER = new HackSelectStarWithColumnACL();
+ QueryContext current = QueryContext.current();
- @Before
- public void setup() throws Exception {
- createTestMetadata();
+ @BeforeEach
+ void setup() {
getTestConfig().setProperty("kylin.query.security.acl-tcr-enabled", "true");
+ prepareBasic();
+ current.setAclInfo(new QueryContext.AclInfo("u1", Sets.newHashSet("g1"), false));
}
- @After
- public void after() {
- cleanupTestMetadata();
+ @AfterAll
+ static void afterAll() {
+ QueryContext.current().close();
}
@Test
- public void testTransform() {
- prepareBasic();
- HackSelectStarWithColumnACL transformer = new HackSelectStarWithColumnACL();
- QueryContext.current().setAclInfo(new QueryContext.AclInfo("u1", Sets.newHashSet("g1"), false));
- String sql = transformer.convert(
- "select * from TEST_KYLIN_FACT t1 join TEST_ORDER t2 on t1.ORDER_ID = t2.ORDER_ID", PROJECT, SCHEMA);
- String expectSQL = "select \"T1\".\"PRICE\", \"T1\".\"ITEM_COUNT\", \"T1\".\"ORDER_ID\", "
- + "\"T2\".\"ORDER_ID\", \"T2\".\"BUYER_ID\", \"T2\".\"TEST_DATE_ENC\" "
- + "from TEST_KYLIN_FACT t1 join TEST_ORDER t2 on t1.ORDER_ID = t2.ORDER_ID";
- assertRoughlyEquals(expectSQL, sql);
+ void testJoin() {
+ // without alias
+ {
+ String sql = "select * from TEST_KYLIN_FACT join TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" "
+ + "join ( select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\" "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
+ // with alias
+ {
+ String sql = "select * from TEST_KYLIN_FACT t1 join TEST_ORDER t2 on t1.ORDER_ID = t2.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"T1\" "
+ + "join ( select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"T2\" "
+ + "on t1.ORDER_ID = t2.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
+ // nested select star
+ {
+ String sql = "select * from (select * from TEST_KYLIN_FACT) t1 join TEST_ORDER t2 "
+ + "on t1.ORDER_ID = t2.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from (" //
+ + "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\""
+ + ") t1 join ( select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\""
+ + ") as \"T2\" on t1.ORDER_ID = t2.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
}
@Test
- public void testTransformColumnStartWithNumberOrKeyword() {
- getTestConfig().setProperty("kylin.query.calcite.extras-props.quoting", "DOUBLE_QUOTE");
- prepareBasic();
+ void testWithSubQuery() {
+ // simple case
+ {
+ String sql = "with test_order as (select * from test_order)\n"
+ + "select * from TEST_KYLIN_FACT join TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "with test_order as (select * from ( "
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\")\n"
+ + "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" "
+ + "join TEST_ORDER on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
+ // some content of with-body reuse with-items
+ {
+ String sql = "with test_order as (select * from test_order)\n"
+ + "select * from TEST_KYLIN_FACT join (select * from TEST_ORDER) TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "with test_order as (select * from ( "
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\")\n"
+ + "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" "
+ + "join (select * from TEST_ORDER) TEST_ORDER on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
+ // all contexts of with-body do not reuse any with-items
+ {
+ String sql = "with test_order as (select * from test_order)\n"
+ + "select * from TEST_KYLIN_FACT join (select * from \"DEFAULT\".\"TEST_ORDER\") TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "with test_order as (select * from ( "
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\")\n"
+ + "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" " //
+ + "join (select * from ( " //
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" "
+ + "from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\") TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
+ // all contexts of with-body reuse with-items
+ {
+ String sql = "with test_order as (select * from test_order), "
+ + "test_kylin_fact as (select * from test_kylin_fact)\n"
+ + "select * from TEST_KYLIN_FACT join (select * from TEST_ORDER) TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "with test_order as (" //
+ + "select * from ( select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" " //
+ + "from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\"), " //
+ + "test_kylin_fact as ("
+ + "select * from ( select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\")\n"
+ + "select * from TEST_KYLIN_FACT join (select * from TEST_ORDER) TEST_ORDER "
+ + "on TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID";
+ Assertions.assertEquals(expected, converted);
+ }
+ }
+
+ @Test
+ void testUnion() {
+ // without outer select
+ {
+ String sql = "select * from test_order union select * from test_order";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from ( " //
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\" "
+ + "union select * from ( " //
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\"";
+ Assertions.assertEquals(expected, converted);
+ }
+ // with outer select
+ {
+ String sql = "select * from (select * from test_order union select * from test_order)";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from (select * from ( " //
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\" "
+ + "union select * from ( " //
+ + "select \"TEST_ORDER\".\"ORDER_ID\", \"TEST_ORDER\".\"BUYER_ID\", "
+ + "\"TEST_ORDER\".\"TEST_DATE_ENC\" from \"DEFAULT\".\"TEST_ORDER\") as \"TEST_ORDER\")";
+ Assertions.assertEquals(expected, converted);
+ }
+ }
+
+ @Test
+ void testInSubQuery() {
+ String sql = "select * from TEST_KYLIN_FACT "
+ + "where ITEM_COUNT in (select ITEM_COUNT from (select * from TEST_KYLIN_FACT) )";
+ String expected = "select * from ( "
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" "
+ + "where ITEM_COUNT in (select ITEM_COUNT from (select * from TEST_KYLIN_FACT) )";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+
+ @Test
+ void testCaseWhen() {
+ {
+ String sql = "select (case when ITEM_COUNT > 0 " //
+ + "then (case when order_id > 0 then order_id else 1 end) " //
+ + "else null end)\n" //
+ + "from TEST_KYLIN_FACT";
+ String expected = "select (case when ITEM_COUNT > 0 " //
+ + "then (case when order_id > 0 then order_id else 1 end) " //
+ + "else null end)\n" //
+ + "from ( select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", " //
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" " //
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\"";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+
+ {
+ String sql = "select * from test_kylin_fact " //
+ + "where case when ITEM_COUNT > 10 then item_count else 0 end";
+ String expected = "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", " //
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" " //
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" "
+ + "where case when ITEM_COUNT > 10 then item_count else 0 end";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+ }
+
+ @Test
+ void testSingleTable() {
+ // without limit
+ {
+ String sql = "select * from \"DEFAULT\".\"TEST_KYLIN_FACT\"";
+ String expected = "select * from ( "
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\"";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+ // with alias
+ {
+ String sql = "select * from test_kylin_fact as test_kylin_fact";
+ String expected = "select * from ( "
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\"";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+ // with limit-offset
+ {
+ String sql = "select * from test_kylin_fact as test_kylin_fact limit 10 offset 2";
+ String expected = "select * from ( "
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\" limit 10 offset 2";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+ // agg
+ {
+ String sql = "select count(*) from \"DEFAULT\".\"TEST_KYLIN_FACT\"";
+ String expected = "select count(*) from ( "
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\"";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ Assertions.assertEquals(expected, converted);
+ }
+ }
+
+ @Test
+ void testKeywordAsColName() {
prepareMore();
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(getTestConfig(), PROJECT);
TableDesc tableDesc = tableMetadataManager.getTableDesc("DEFAULT.TEST_KYLIN_FACT");
@@ -87,28 +307,24 @@ public class HackSelectStarWithColumnACLTest extends NLocalFileMetadataTestCase
colWithKeyword.setDatatype("date");
colWithKeyword.setName("YEAR");
- ArrayList<ColumnDesc> columnDescs = Lists.newArrayList(columns);
+ List<ColumnDesc> columnDescs = Lists.newArrayList(columns);
columnDescs.add(colStartsWithNumber);
columnDescs.add(colWithKeyword);
tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[0]));
tableMetadataManager.updateTableDesc(tableDesc);
- HackSelectStarWithColumnACL transformer = new HackSelectStarWithColumnACL();
- QueryContext.current().setAclInfo(new QueryContext.AclInfo("u1", Sets.newHashSet("g1"), false));
- String transformed = transformer.convert("select * from TEST_KYLIN_FACT", PROJECT, SCHEMA);
- String expected = "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", " //
- + "\"TEST_KYLIN_FACT\".\"PRICE\", " //
- + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\", " //
- + "\"TEST_KYLIN_FACT\".\"2D\", " //
- + "\"TEST_KYLIN_FACT\".\"YEAR\" " //
- + "from TEST_KYLIN_FACT";
- Assert.assertEquals(expected, transformed);
+
+ String sql = "select * from TEST_KYLIN_FACT";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", \"TEST_KYLIN_FACT\".\"PRICE\", "
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\", \"TEST_KYLIN_FACT\".\"2D\", \"TEST_KYLIN_FACT\".\"YEAR\" "
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\"";
+ Assertions.assertEquals(expected, converted);
}
@Test
- public void testTransformColumnStartWithNumberOrKeyword2() {
- getTestConfig().setProperty("kylin.query.calcite.extras-props.quoting", "BACK_TICK");
- prepareBasic();
+ void testColumnNameStartsWithNumber() {
prepareMore();
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(getTestConfig(), PROJECT);
TableDesc tableDesc = tableMetadataManager.getTableDesc("DEFAULT.TEST_KYLIN_FACT");
@@ -123,86 +339,29 @@ public class HackSelectStarWithColumnACLTest extends NLocalFileMetadataTestCase
colWithKeyword.setDatatype("date");
colWithKeyword.setName("YEAR");
- ArrayList<ColumnDesc> columnDescs = Lists.newArrayList(columns);
+ List<ColumnDesc> columnDescs = Lists.newArrayList(columns);
columnDescs.add(colStartsWithNumber);
columnDescs.add(colWithKeyword);
tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[0]));
tableMetadataManager.updateTableDesc(tableDesc);
- HackSelectStarWithColumnACL transformer = new HackSelectStarWithColumnACL();
- QueryContext.current().setAclInfo(new QueryContext.AclInfo("u1", Sets.newHashSet("g1"), false));
- String transformed = transformer.convert("select * from TEST_KYLIN_FACT", PROJECT, SCHEMA);
- String expected = "select `TEST_KYLIN_FACT`.`ORDER_ID`, " //
- + "`TEST_KYLIN_FACT`.`PRICE`, " //
- + "`TEST_KYLIN_FACT`.`ITEM_COUNT`, " //
- + "`TEST_KYLIN_FACT`.`2D`, " //
- + "`TEST_KYLIN_FACT`.`YEAR` " //
- + "from TEST_KYLIN_FACT";
- Assert.assertEquals(expected, transformed);
- }
- @Test
- public void testExplainSyntax() {
- HackSelectStarWithColumnACL transformer = new HackSelectStarWithColumnACL();
- String sql = "explain plan for select * from t";
- assertRoughlyEquals(sql, transformer.convert("explain plan for select * from t", PROJECT, SCHEMA));
- }
-
- @Test
- public void testGetNewSelectClause() {
- prepareBasic();
- final String sql = "select * from TEST_KYLIN_FACT t1 join TEST_ORDER t2 on t1.ORDER_ID = t2.ORDER_ID ";
- final SqlNode sqlNode = getSqlNode(sql);
- QueryContext.AclInfo aclInfo = new QueryContext.AclInfo("u1", Sets.newHashSet("g1"), false);
- String newSelectClause = HackSelectStarWithColumnACL.getNewSelectClause(sqlNode, PROJECT, SCHEMA, aclInfo);
- String expect = "\"T1\".\"PRICE\", \"T1\".\"ITEM_COUNT\", \"T1\".\"ORDER_ID\", "
- + "\"T2\".\"ORDER_ID\", \"T2\".\"BUYER_ID\", \"T2\".\"TEST_DATE_ENC\"";
- assertRoughlyEquals(expect, newSelectClause);
-
- AclTCR empty = new AclTCR();
- empty.setTable(new AclTCR.Table());
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), PROJECT);
- manager.updateAclTCR(empty, "u1", true);
- manager.updateAclTCR(empty, "g1", false);
-
- try {
- HackSelectStarWithColumnACL.getNewSelectClause(sqlNode, PROJECT, SCHEMA, aclInfo);
- } catch (Exception e) {
- Assert.assertEquals(NoAuthorizedColsError.class, e.getClass());
- }
+ String sql = "select * from TEST_KYLIN_FACT";
+ String converted = TRANSFORMER.convert(sql, PROJECT, SCHEMA);
+ String expected = "select * from ( " //
+ + "select \"TEST_KYLIN_FACT\".\"ORDER_ID\", " //
+ + "\"TEST_KYLIN_FACT\".\"PRICE\", " //
+ + "\"TEST_KYLIN_FACT\".\"ITEM_COUNT\", " //
+ + "\"TEST_KYLIN_FACT\".\"2D\", " //
+ + "\"TEST_KYLIN_FACT\".\"YEAR\" " //
+ + "from \"DEFAULT\".\"TEST_KYLIN_FACT\") as \"TEST_KYLIN_FACT\"";
+ Assertions.assertEquals(expected, converted);
}
@Test
- public void testGetColsCanAccess() {
-
- final String sql = "select * from TEST_KYLIN_FACT t1 join TEST_ORDER t2 on t1.ORDER_ID = t2.ORDER_ID";
- final SqlNode sqlNode = getSqlNode(sql);
- QueryContext.AclInfo aclInfo = new QueryContext.AclInfo("u1", Sets.newHashSet("g1"), false);
- List<String> colsCanAccess = HackSelectStarWithColumnACL.getColsCanAccess(sqlNode, PROJECT, SCHEMA, aclInfo);
- Assert.assertEquals(0, colsCanAccess.size());
-
- prepareBasic();
- colsCanAccess = HackSelectStarWithColumnACL.getColsCanAccess(sqlNode, PROJECT, SCHEMA, aclInfo);
- Assert.assertEquals(6, colsCanAccess.size());
- }
-
- private SqlNode getSqlNode(String sql) {
- SqlNode sqlNode;
- try {
- sqlNode = CalciteParser.parse(sql);
- } catch (SqlParseException e) {
- throw new RuntimeException("Failed to parse SQL \'" + sql + "\', please make sure the SQL is valid");
- }
- return sqlNode;
- }
-
- private void assertRoughlyEquals(String expect, String actual) {
- String[] expectSplit = expect.split("\\s+");
- String[] actualSplit = actual.split("\\s+");
- Arrays.sort(expectSplit);
- Arrays.sort(actualSplit);
-
- Assert.assertArrayEquals(expectSplit, actualSplit);
+ void testExplainSyntax() {
+ String sql = "explain plan for select * from t";
+ Assertions.assertEquals(sql, TRANSFORMER.convert("explain plan for select * from t", PROJECT, SCHEMA));
}
private void prepareMore() {
@@ -241,7 +400,7 @@ public class HackSelectStarWithColumnACLTest extends NLocalFileMetadataTestCase
AclTCR.Table g1t1 = new AclTCR.Table();
AclTCR.ColumnRow g1cr1 = new AclTCR.ColumnRow();
AclTCR.Column g1c1 = new AclTCR.Column();
- g1c1.addAll(Arrays.asList("ORDER_ID"));
+ g1c1.add("ORDER_ID");
g1cr1.setColumn(g1c1);
g1t1.put("DEFAULT.TEST_KYLIN_FACT", g1cr1);
g1a1.setTable(g1t1);
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
index 5b91a8cef2..669fdde19c 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/PushDownUtilTest.java
@@ -20,12 +20,17 @@ package org.apache.kylin.query.util;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.exception.ServerErrorCode;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.TestUtils;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.acl.AclTCR;
+import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -34,65 +39,86 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.util.AclPermissionUtil;
+import org.apache.kylin.util.MetadataTestUtils;
+import org.glassfish.jersey.internal.guava.Sets;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.springframework.security.authentication.TestingAuthenticationToken;
+import org.springframework.security.core.context.SecurityContextHolder;
-public class PushDownUtilTest extends NLocalFileMetadataTestCase {
+@MetadataInfo
+class PushDownUtilTest {
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
+ private static void setAdminAuthentication() {
+ TestingAuthenticationToken auth = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
+ SecurityContextHolder.getContext().setAuthentication(auth);
}
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
+ private static void setNormalAuthenticationAndAclTcr() {
+ TestingAuthenticationToken auth = new TestingAuthenticationToken("KYLIN", "KYLIN", Constant.ROLE_MODELER);
+ SecurityContextHolder.getContext().setAuthentication(auth);
+ AclTCRManager manager = AclTCRManager.getInstance(TestUtils.getTestConfig(), "default");
+ manager.updateAclTCR(new AclTCR(), "KYLIN", true);
+ }
+
+ private static QueryContext.AclInfo prepareRoleModelerAclInfo(String project) {
+ Set<String> groups = Sets.newHashSet();
+ groups.add(Constant.ROLE_MODELER);
+ return AclPermissionUtil.createAclInfo(project, groups);
+ }
+
+ private static QueryContext.AclInfo prepareRoleAdminAclInfo(String project) {
+ Set<String> groups = Sets.newHashSet();
+ groups.add("admin");
+ groups.add(Constant.ROLE_ADMIN);
+ return AclPermissionUtil.createAclInfo(project, groups);
}
@Test
- public void testTryForcePushDown() {
+ void testTryForcePushDown() {
try {
QueryParams queryParams = new QueryParams();
queryParams.setProject("default");
queryParams.setSelect(true);
queryParams.setForcedToPushDown(true);
PushDownUtil.tryIterQuery(queryParams);
- Assert.fail();
+ Assertions.fail();
} catch (Exception e) {
- Assert.assertTrue(e instanceof KylinException);
- Assert.assertEquals(ServerErrorCode.SPARK_FAILURE.toErrorCode(), ((KylinException) e).getErrorCode());
+ Assertions.assertTrue(e instanceof KylinException);
+ Assertions.assertEquals(ServerErrorCode.SPARK_FAILURE.toErrorCode(), ((KylinException) e).getErrorCode());
}
}
@Test
- public void testTryWithPushDownDisable() {
+ void testTryWithPushDownDisable() {
try {
- overwriteSystemProp("kylin.query.pushdown-enabled", "false");
+ String project = "default";
+ MetadataTestUtils.updateProjectConfig(project, "kylin.query.pushdown-enabled", "false");
QueryParams queryParams = new QueryParams();
queryParams.setProject("default");
queryParams.setSelect(true);
queryParams.setForcedToPushDown(true);
PushDownUtil.tryIterQuery(queryParams);
- Assert.fail();
+ Assertions.fail();
} catch (Exception e) {
- Assert.assertTrue(e instanceof KylinException);
- Assert.assertEquals(QueryErrorCode.INVALID_PARAMETER_PUSH_DOWN.toErrorCode(),
+ Assertions.assertTrue(e instanceof KylinException);
+ Assertions.assertEquals(QueryErrorCode.INVALID_PARAMETER_PUSH_DOWN.toErrorCode(),
((KylinException) e).getErrorCode());
}
}
@Test
- public void testBacktickQuote() {
+ void testBacktickQuote() {
String table = "db.table";
- Assert.assertEquals("`db`.`table`", String.join(".", PushDownUtil.backtickQuote(table.split("\\."))));
+ Assertions.assertEquals("`db`.`table`", String.join(".", PushDownUtil.backtickQuote(table.split("\\."))));
}
@Test
- public void testMassagePushDownSql() {
+ void testMassagePushDownSql() {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
+ try (KylinConfig.SetAndUnsetThreadLocalConfig ignored = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
config.setProperty("kylin.query.pushdown.converter-class-names",
SparkSQLFunctionConverter.class.getCanonicalName());
String sql = "SELECT \"Z_PROVDASH_UM_ED\".\"GENDER\" AS \"GENDER\",\n"
@@ -105,25 +131,25 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`,\n"
+ "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
+ "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` `Z_PROVDASH_UM_ED`";
- Assert.assertEquals(expectedSql, massagedSql);
+ Assertions.assertEquals(expectedSql, massagedSql);
}
}
@Test
- public void testMassagePushDownSqlWithDoubleQuote() {
+ void testMassagePushDownSqlWithDoubleQuote() {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
String sql = "select '''',trans_id from test_kylin_fact where LSTG_FORMAT_NAME like '%''%' group by trans_id limit 2;";
QueryParams queryParams = new QueryParams("", sql, "default", false);
queryParams.setKylinConfig(config);
String massagedSql = PushDownUtil.massagePushDownSql(queryParams);
String expectedSql = "select '\\'', `TRANS_ID` from `TEST_KYLIN_FACT` where `LSTG_FORMAT_NAME` like '%\\'%' group by `TRANS_ID` limit 2";
- Assert.assertEquals(expectedSql, massagedSql);
+ Assertions.assertEquals(expectedSql, massagedSql);
}
@Test
- public void testMassagePushDownSqlWithDialectConverter() {
+ void testMassagePushDownSqlWithDialectConverter() {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
- try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
+ try (KylinConfig.SetAndUnsetThreadLocalConfig ignored = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
config.setProperty("kylin.query.pushdown.converter-class-names",
"org.apache.kylin.query.util.DialectConverter,org.apache.kylin.source.adhocquery.DoubleQuotePushDownConverter,"
+ SparkSQLFunctionConverter.class.getCanonicalName());
@@ -138,141 +164,182 @@ public class PushDownUtilTest extends NLocalFileMetadataTestCase {
String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`, "
+ "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
+ "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` AS `Z_PROVDASH_UM_ED`\n" + "LIMIT 1";
- Assert.assertEquals(expectedSql, massagedSql);
+ Assertions.assertEquals(expectedSql, massagedSql);
}
}
@Test
- public void testReplaceDoubleQuoteToSingle() {
+ void testReplaceDoubleQuoteToSingle() {
String sql = "select ab from table where aa = '' and bb = '''as''n'''";
String resSql = "select ab from table where aa = '' and bb = '\\'as\\'n\\''";
- Assert.assertEquals(resSql, PushDownUtil.replaceEscapedQuote(sql));
+ Assertions.assertEquals(resSql, PushDownUtil.replaceEscapedQuote(sql));
}
@Test
- public void testGenerateFlatTableSql() {
- String project = "default";
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
- String expected = "SELECT\n" //
- + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
- + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
- + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
- + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
- + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
- + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
- + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
- + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
- + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
- + "WHERE\n" //
- + "1 = 1";
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, false));
+ void testGenerateFlatTableSql() {
+ try (QueryContext context = QueryContext.current()) {
+ String project = "default";
+ setNormalAuthenticationAndAclTcr();
+ context.setAclInfo(prepareRoleModelerAclInfo(project));
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1";
+ Assertions.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, false));
+ }
}
@Test
- public void testGenerateFlatTableSqlWithCCJoin() {
- String project = "default";
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
- updateModelToAddCC(project, model);
- // change join condition
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
- modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
- List<JoinTableDesc> joinTables = copyForWrite.getJoinTables();
- TableRef rootTableRef = copyForWrite.getRootFactTable();
- TblColRef cc1 = rootTableRef.getColumn("CC1");
- JoinDesc join = joinTables.get(0).getJoin();
- join.setForeignKeyColumns(new TblColRef[] { cc1 });
- join.setForeignKey(new String[] { "TEST_BANK_INCOME.CC1" });
- });
- return null;
- }, project);
- String expected = "SELECT\n" //
- + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
- + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
- + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
- + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
- + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
- + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
- + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
- + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
- + "ON SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n"
- + "WHERE\n" //
- + "1 = 1";
- NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
+ void testGenerateFlatTableSqlOfRoleModeler() {
+ try (QueryContext context = QueryContext.current()) {
+ String project = "default";
+ setAdminAuthentication();
+ context.setAclInfo(prepareRoleAdminAclInfo(project));
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1";
+ Assertions.assertEquals(expected, PushDownUtil.generateFlatTableSql(model, false));
+ }
+ }
+ @Test
+ void testGenerateFlatTableSqlWithCCJoin() {
+ try (QueryContext context = QueryContext.current()) {
+ String project = "default";
+ setNormalAuthenticationAndAclTcr();
+ context.setAclInfo(prepareRoleModelerAclInfo(project));
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change join condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ List<JoinTableDesc> joinTables = copyForWrite.getJoinTables();
+ TableRef rootTableRef = copyForWrite.getRootFactTable();
+ TblColRef cc1 = rootTableRef.getColumn("CC1");
+ JoinDesc join = joinTables.get(0).getJoin();
+ join.setForeignKeyColumns(new TblColRef[] { cc1 });
+ join.setForeignKey(new String[] { "TEST_BANK_INCOME.CC1" });
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n"
+ + "WHERE\n" //
+ + "1 = 1";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assertions.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
+ }
}
@Test
- public void testGenerateFlatTableSqlWithFilterCondition() {
- String project = "default";
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
- updateModelToAddCC(project, model);
- // change filter condition
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
- modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
- copyForWrite.setFilterCondition(
- "SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china'");
- });
- return null;
- }, project);
- String expected = "SELECT\n" //
- + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
- + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
- + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
- + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
- + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
- + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
- + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
- + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
- + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
- + "WHERE\n" //
- + "1 = 1\n" //
- + " AND (SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china')";
- NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
+ void testGenerateFlatTableSqlWithFilterCondition() {
+ try (QueryContext context = QueryContext.current()) {
+ String project = "default";
+ setNormalAuthenticationAndAclTcr();
+ context.setAclInfo(prepareRoleModelerAclInfo(project));
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change filter condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ copyForWrite.setFilterCondition(
+ "SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china'");
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1\n" //
+ + " AND (SUBSTRING(\"TEST_BANK_INCOME\".\"COUNTRY\", 0, 4) = 'china' and cc1 = 'china')";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assertions.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
+ }
}
@Test
- public void testGenerateFlatTableSqlWithSpecialFunctions() {
- String project = "default";
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
- updateModelToAddCC(project, model);
- // change filter condition
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
- modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
- copyForWrite.setFilterCondition("timestampadd(day, 1, current_date) = '2012-01-01' and cc1 = 'china'");
- });
- return null;
- }, project);
- String expected = "SELECT\n" //
- + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
- + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
- + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
- + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
- + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
- + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
- + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
- + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
- + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
- + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
- + "WHERE\n" //
- + "1 = 1\n" //
- + " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and cc1 = 'china')";
- NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
- Assert.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
+ void testGenerateFlatTableSqlWithSpecialFunctions() {
+ try (QueryContext context = QueryContext.current()) {
+ String project = "default";
+ setNormalAuthenticationAndAclTcr();
+ context.setAclInfo(prepareRoleModelerAclInfo(project));
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel model = modelManager.getDataModelDescByAlias("test_bank");
+ updateModelToAddCC(project, model);
+ // change filter condition
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ NDataModelManager modelMgr = NDataModelManager.getInstance(kylinConfig, project);
+ modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
+ copyForWrite
+ .setFilterCondition("timestampadd(day, 1, current_date) = '2012-01-01' and cc1 = 'china'");
+ });
+ return null;
+ }, project);
+ String expected = "SELECT\n" //
+ + "\"TEST_BANK_INCOME\".\"COUNTRY\" as \"TEST_BANK_INCOME_COUNTRY\"\n"
+ + ", \"TEST_BANK_INCOME\".\"INCOME\" as \"TEST_BANK_INCOME_INCOME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"NAME\" as \"TEST_BANK_INCOME_NAME\"\n"
+ + ", \"TEST_BANK_INCOME\".\"DT\" as \"TEST_BANK_INCOME_DT\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"COUNTRY\" as \"TEST_BANK_LOCATION_COUNTRY\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"OWNER\" as \"TEST_BANK_LOCATION_OWNER\"\n"
+ + ", \"TEST_BANK_LOCATION\".\"LOCATION\" as \"TEST_BANK_LOCATION_LOCATION\"\n"
+ + "FROM \"DEFAULT\".\"TEST_BANK_INCOME\" as \"TEST_BANK_INCOME\"\n"
+ + "INNER JOIN \"DEFAULT\".\"TEST_BANK_LOCATION\" as \"TEST_BANK_LOCATION\"\n"
+ + "ON \"TEST_BANK_INCOME\".\"COUNTRY\" = \"TEST_BANK_LOCATION\".\"COUNTRY\"\n" //
+ + "WHERE\n" //
+ + "1 = 1\n" //
+ + " AND (TIMESTAMPADD(day, 1, current_date) = '2012-01-01' and cc1 = 'china')";
+ NDataModel updatedModel = modelManager.getDataModelDesc(model.getUuid());
+ Assertions.assertEquals(expected, PushDownUtil.generateFlatTableSql(updatedModel, false));
+ }
}
private void updateModelToAddCC(String project, NDataModel model) {
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
index 52943ecd2e..cb1e1d3547 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.engine.spark.utils.SparkJobFactoryUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.cube.model.NDataLayout;
@@ -38,8 +39,6 @@ import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import com.clearspring.analytics.util.Lists;
-
import lombok.val;
import lombok.extern.slf4j.Slf4j;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
index 6f3b1cf5ed..12b91f0932 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
@@ -45,6 +45,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.guava30.shaded.common.cache.Cache;
import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ISourceAware;
@@ -64,8 +65,6 @@ import org.apache.spark.sql.internal.SQLConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.clearspring.analytics.util.Lists;
-
import lombok.val;
public class NSparkMetadataExplorer implements ISourceMetadataExplorer, ISampleDataDeployer, Serializable {
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkMergingJobTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkMergingJobTest.java
index e41fa4e652..7a09678bfc 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkMergingJobTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkMergingJobTest.java
@@ -18,14 +18,16 @@
package org.apache.kylin.engine.spark.job;
-import com.clearspring.analytics.util.Lists;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
-import lombok.val;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.engine.spark.IndexDataConstructor;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.engine.spark.merger.AfterMergeOrRefreshResourceMerger;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -47,9 +49,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
+import lombok.val;
public class NSparkMergingJobTest extends NLocalWithSparkSessionTest {
(kylin) 04/35: KYLIN-5796 Support databinder auto grow collections limit
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e824503ecaeca7f657de11bf12113ab5fd1eadb3
Author: haocheni <ha...@qq.com>
AuthorDate: Tue Aug 29 14:59:55 2023 +0800
KYLIN-5796 Support databinder auto grow collections limit
---
.../kylin/rest/controller/NBasicController.java | 14 ++++++++
.../kylin/rest/aspect/InsensitiveNameAspect.java | 3 ++
.../org/apache/kylin/common/KylinConfigBase.java | 15 +++++++++
.../kylin/common/exception/ServerErrorCode.java | 1 +
.../org/apache/kylin/common/msg/CnMessage.java | 10 ++++++
.../java/org/apache/kylin/common/msg/Message.java | 9 +++++
.../resources/kylin_errorcode_conf_en.properties | 1 +
.../resources/kylin_errorcode_conf_zh.properties | 1 +
src/metadata-server/pom.xml | 5 +++
.../controller/NProjectControllerKylinTest.java | 17 ++++------
.../rest/controller/NProjectControllerV2Test.java | 17 ++++------
.../rest/controller/NUserGroupControllerTest.java | 38 ++++++++++------------
.../controller/NUserGroupControllerV2Test.java | 22 +++++--------
.../rest/controller/OpenAclTCRControllerTest.java | 15 +++------
.../controller/open/OpenAclTCRControllerTest.java | 8 +++--
.../open/OpenUserGroupControllerTest.java | 14 ++++----
.../rest/controller/NQueryMetaControllerTest.java | 2 ++
.../rest/controller/SparkSourceControllerTest.java | 12 ++++---
18 files changed, 127 insertions(+), 77 deletions(-)
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index e88ce19d72..61854b7436 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -24,6 +24,7 @@ import static org.apache.kylin.common.exception.ServerErrorCode.EMPTY_PROJECT_NA
import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_CONNECT_CATALOG;
import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_DOWNLOAD_FILE;
import static org.apache.kylin.common.exception.ServerErrorCode.INCORRECT_PROJECT_MODE;
+import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_DATA_BINDER_AUTO_GROW_COLLECTION_LIMIT;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER;
import static org.apache.kylin.common.exception.ServerErrorCode.LOW_LEVEL_LICENSE;
import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION;
@@ -110,6 +111,7 @@ import org.apache.kylin.rest.util.PagingUtil;
import org.apache.kylin.util.DataRangeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.InvalidPropertyException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.HttpEntity;
@@ -128,7 +130,9 @@ import org.springframework.transaction.CannotCreateTransactionException;
import org.springframework.util.StreamUtils;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.MissingServletRequestParameterException;
+import org.springframework.web.bind.WebDataBinder;
import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.InitBinder;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.client.RequestCallback;
@@ -184,6 +188,10 @@ public class NBasicController {
if (cause instanceof CannotCreateTransactionException) {
kylinException = new KylinException(FAILED_CONNECT_CATALOG, msg.getConnectDatabaseError(), false);
}
+ if (cause instanceof InvalidPropertyException && cause.getCause() instanceof IndexOutOfBoundsException) {
+ kylinException = new KylinException(INVALID_DATA_BINDER_AUTO_GROW_COLLECTION_LIMIT,
+ msg.getInvalidDataBinderAutoGrowCollectionLimit(), false);
+ }
if (cause instanceof KylinException) {
kylinException = (KylinException) cause;
}
@@ -679,4 +687,10 @@ public class NBasicController {
}
}
}
+
+ @InitBinder
+ private void initBinder(WebDataBinder binder) {
+ int autoGrowCollectionLimit = KylinConfig.getInstanceFromEnv().getDataBinderAutoGrowCollectionLimit();
+ binder.setAutoGrowCollectionLimit(autoGrowCollectionLimit);
+ }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
index cfc43bf7b2..8d54a85807 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
@@ -125,6 +125,9 @@ public class InsensitiveNameAspect {
for (int i = 0; i < parameterNames.length; i++) {
if (parameterTypes[i] == String.class && Objects.equals("project", parameterNames[i])) {
projectName = (String) args[i];
+ if (StringUtils.isBlank(projectName)) {
+ continue;
+ }
NProjectManager projectManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
ProjectInstance projectInstance = projectManager.getProject(projectName);
if (projectInstance != null) {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 9bce3f735a..3de55bf84e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -53,6 +53,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -3073,6 +3074,20 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.web.session.jdbc-encode-enabled", FALSE));
}
+ public int getDataBinderAutoGrowCollectionLimit() {
+ String defaultValue = "256";
+ String value = getOptional("kylin.web.data-binder.auto-grow-collection-limit", defaultValue);
+
+ if (!NumberUtils.isDigits(value)) {
+ return Integer.parseInt(defaultValue);
+ }
+ int intValue = Integer.parseInt(value);
+ if (intValue <= 0) {
+ return Integer.parseInt(defaultValue);
+ }
+ return intValue;
+ }
+
public String getSpringStoreType() {
return getOptional("spring.session.store-type", "");
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
index 881a6c1f7f..68aa6bcd3f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
@@ -281,6 +281,7 @@ public enum ServerErrorCode implements ErrorCodeSupplier {
SPARK_FAILURE("KE-010038002"), //
CONFIG_NONEXIST_PROJECT("KE-010038003"), //
CONFIG_NONEXIST_MODEL("KE-010038004"), //
+ INVALID_DATA_BINDER_AUTO_GROW_COLLECTION_LIMIT("KE-010038005"), //
// 10039XXX jdbc source
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index 09c3925756..5d15f7042c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -1725,4 +1725,14 @@ public class CnMessage extends Message {
public String getIllegalNegative(String parameter) {
return String.format(PARAMETER_MUST_BE_POSITIVE_NUMBER, parameter);
}
+
+ @Override
+ public String getInvalidDataBinderAutoGrowCollectionLimit() {
+ return "操作失败，请调大参数 kylin.web.data-binder.auto-grow-collection-limit 的值后再试。";
+ }
+
+ @Override
+ public String getRedisInitFailed() {
+ return "Redis初始化失败";
+ }
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 3dff08d642..e43eece2e5 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -1628,4 +1628,13 @@ public class Message {
}
+ public String getInvalidDataBinderAutoGrowCollectionLimit() {
+ return "Operation failed, please increase the value of "
+ + "kylin.web.data-binder.auto-grow-collection-limit and try again.";
+ }
+
+ public String getRedisInitFailed() {
+ return "Redis init failed";
+ }
+
}
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
index b82744456d..02e3f7f830 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
@@ -273,6 +273,7 @@ KE-010038001=System Profile Abnormal Data
KE-010038002=Spark Failure
KE-010038003=Non Exist Project
KE-010038004=Non Exist Model
+KE-010038005=invalid data binder auto grow collection limit
KE-010039001=Invalid Connection Info
# cache
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
index 9f04bec06f..d8fbfc42ed 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
@@ -271,6 +271,7 @@ KE-010038001=系统配置数据异常
KE-010038002=Spark出错
KE-010038003=不存在的项目
KE-010038004=不存在的模型
+KE-010038005=无效的表单数组自动增长限制
KE-010039001=连接信息有误
# cache
diff --git a/src/metadata-server/pom.xml b/src/metadata-server/pom.xml
index 151ab4df97..28a360d290 100644
--- a/src/metadata-server/pom.xml
+++ b/src/metadata-server/pom.xml
@@ -89,6 +89,11 @@
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerKylinTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerKylinTest.java
index 779e79dae2..e44d8683ac 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerKylinTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerKylinTest.java
@@ -22,15 +22,14 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.v2.NProjectControllerKylin;
import org.apache.kylin.rest.service.ProjectService;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -44,13 +43,11 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+@MetadataInfo(onlyProps = true)
public class NProjectControllerKylinTest {
private MockMvc mockMvc;
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@Mock
private ProjectService projectService;
@@ -59,7 +56,7 @@ public class NProjectControllerKylinTest {
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
- @Before
+ @BeforeEach
public void setup() {
MockitoAnnotations.initMocks(this);
mockMvc = MockMvcBuilders.standaloneSetup(nProjectControllerKylin)
@@ -68,7 +65,7 @@ public class NProjectControllerKylinTest {
SecurityContextHolder.getContext().setAuthentication(authentication);
}
- @After
+ @AfterEach
public void tearDown() {
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerV2Test.java
index 2b25ca4285..28999ee180 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerV2Test.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NProjectControllerV2Test.java
@@ -23,15 +23,14 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.v2.NProjectControllerV2;
import org.apache.kylin.rest.service.ProjectService;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -45,13 +44,11 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+@MetadataInfo(onlyProps = true)
public class NProjectControllerV2Test {
private MockMvc mockMvc;
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@Mock
private ProjectService projectService;
@@ -60,7 +57,7 @@ public class NProjectControllerV2Test {
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
- @Before
+ @BeforeEach
public void setup() {
MockitoAnnotations.initMocks(this);
mockMvc = MockMvcBuilders.standaloneSetup(nProjectControllerV2).defaultRequest(MockMvcRequestBuilders.get("/"))
@@ -69,7 +66,7 @@ public class NProjectControllerV2Test {
SecurityContextHolder.getContext().setAuthentication(authentication);
}
- @After
+ @AfterEach
public void tearDown() {
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerTest.java
index 119f49253b..261c66ec82 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerTest.java
@@ -18,15 +18,17 @@
package org.apache.kylin.rest.controller;
+import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_JSON;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY;
import static org.apache.kylin.rest.constant.Constant.ROLE_ADMIN;
-import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_JSON;
import java.util.Arrays;
import java.util.List;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.metadata.usergroup.UserGroup;
@@ -35,12 +37,11 @@ import org.apache.kylin.rest.request.UserGroupRequest;
import org.apache.kylin.rest.response.UserGroupResponseKI;
import org.apache.kylin.rest.service.AclTCRService;
import org.apache.kylin.rest.service.NUserGroupService;
-import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -61,10 +62,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.val;
+@MetadataInfo
public class NUserGroupControllerTest {
private MockMvc mockMvc;
@@ -78,15 +79,12 @@ public class NUserGroupControllerTest {
@Mock
private UserService userService;
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@InjectMocks
private NUserGroupController nUserGroupController = Mockito.spy(new NUserGroupController());
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", ROLE_ADMIN);
- @Before
+ @BeforeEach
public void setup() {
FilterProvider filterProvider = new SimpleFilterProvider().addFilter("passwordFilter",
SimpleBeanPropertyFilter.serializeAllExcept("password", "defaultPassword"));
@@ -101,7 +99,7 @@ public class NUserGroupControllerTest {
SecurityContextHolder.getContext().setAuthentication(authentication);
}
- @After
+ @AfterEach
public void tearDown() {
}
@@ -155,19 +153,19 @@ public class NUserGroupControllerTest {
}
@Test
- public void testAddEmptyGroup() throws Exception {
- thrown.expect(KylinException.class);
- thrown.expectMessage(REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY.getMsg("group_name"));
- UserGroupRequest request = new UserGroupRequest();
- request.setGroupName("");
- nUserGroupController.addUserGroup(request);
+ public void testAddEmptyGroup() {
+ Assertions.assertThrows(KylinException.class, () -> {
+ UserGroupRequest request = new UserGroupRequest();
+ request.setGroupName("");
+ nUserGroupController.addUserGroup(request);
+ }, REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY.getMsg("group_name"));
}
- @Test(expected = KylinException.class)
+ @Test
public void testAddIllegalGroupName() throws Exception {
UserGroupRequest request = new UserGroupRequest();
request.setGroupName(".hhhh");
- nUserGroupController.addUserGroup(request);
+ Assertions.assertThrows(KylinException.class, () -> nUserGroupController.addUserGroup(request));
}
@Test
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java
index 7727ea5614..40524b1949 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NUserGroupControllerV2Test.java
@@ -21,15 +21,14 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.List;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.rest.controller.v2.NUserGroupControllerV2;
import org.apache.kylin.rest.service.NUserGroupService;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -44,10 +43,10 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.accept.ContentNegotiationManager;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
+import io.kyligence.kap.metadata.user.ManagedUser;
import lombok.val;
+@MetadataInfo(onlyProps = true)
public class NUserGroupControllerV2Test {
private MockMvc mockMvc;
@@ -55,15 +54,12 @@ public class NUserGroupControllerV2Test {
@Mock
private NUserGroupService userGroupService;
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@InjectMocks
private NUserGroupControllerV2 nUserGroupControllerV2 = Mockito.spy(new NUserGroupControllerV2());
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
- @Before
+ @BeforeEach
public void setup() {
MockitoAnnotations.initMocks(this);
ContentNegotiationManager contentNegotiationManager = new ContentNegotiationManager();
@@ -73,7 +69,7 @@ public class NUserGroupControllerV2Test {
SecurityContextHolder.getContext().setAuthentication(authentication);
}
- @After
+ @AfterEach
public void tearDown() {
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/OpenAclTCRControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/OpenAclTCRControllerTest.java
index 14b227aa9c..ec29ce066e 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/OpenAclTCRControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/OpenAclTCRControllerTest.java
@@ -32,11 +32,9 @@ import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.rest.controller.open.OpenAclTCRController;
import org.apache.kylin.rest.request.AclTCRRequest;
import org.apache.kylin.rest.service.AclTCRService;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -73,12 +71,9 @@ public class OpenAclTCRControllerTest extends NLocalFileMetadataTestCase {
@InjectMocks
private OpenAclTCRController openAclTCRController = Mockito.spy(new OpenAclTCRController());
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
private static final String APPLICATION_JSON_PUBLIC = HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON;
- @Before
+ @BeforeEach
public void setup() throws IOException {
MockitoAnnotations.initMocks(this);
mockMvc = MockMvcBuilders.standaloneSetup(openAclTCRController).defaultRequest(MockMvcRequestBuilders.get("/"))
@@ -89,7 +84,7 @@ public class OpenAclTCRControllerTest extends NLocalFileMetadataTestCase {
createTestMetadata();
}
- @After
+ @AfterEach
public void teardown() {
cleanupTestMetadata();
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenAclTCRControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenAclTCRControllerTest.java
index 8b182e9f7d..1823dd91ce 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenAclTCRControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenAclTCRControllerTest.java
@@ -26,14 +26,15 @@ import java.util.List;
import java.util.Map;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.service.AccessService;
import org.apache.kylin.rest.service.IUserGroupService;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.rest.request.AclTCRRequest;
import org.apache.kylin.rest.service.AclTCRService;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -56,6 +57,7 @@ import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
/**
* created by lee
**/
+@MetadataInfo
public class OpenAclTCRControllerTest {
@Mock
private UserService userService;
@@ -74,7 +76,7 @@ public class OpenAclTCRControllerTest {
private MockMvc mockMvc;
- @Before
+ @BeforeEach
public void setup() {
FilterProvider filterProvider = new SimpleFilterProvider().addFilter("passwordFilter",
SimpleBeanPropertyFilter.serializeAllExcept("password", "defaultPassword"));
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenUserGroupControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenUserGroupControllerTest.java
index f5a6e6a8ca..b3a8a7f425 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenUserGroupControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenUserGroupControllerTest.java
@@ -25,15 +25,17 @@ import java.util.List;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.response.DataResult;
-import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.controller.NUserGroupController;
import org.apache.kylin.rest.request.UpdateGroupRequest;
import org.apache.kylin.rest.request.UserGroupRequest;
+import org.apache.kylin.rest.response.DataResult;
+import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.service.NUserGroupService;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -53,10 +55,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.val;
+@MetadataInfo(onlyProps = true)
public class OpenUserGroupControllerTest {
private MockMvc mockMvc;
@@ -71,7 +73,7 @@ public class OpenUserGroupControllerTest {
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
- @Before
+ @BeforeEach
public void setup() {
FilterProvider filterProvider = new SimpleFilterProvider().addFilter("passwordFilter",
SimpleBeanPropertyFilter.serializeAllExcept("password", "defaultPassword"));
diff --git a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryMetaControllerTest.java b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryMetaControllerTest.java
index e2d22d1087..f445a12197 100644
--- a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryMetaControllerTest.java
+++ b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryMetaControllerTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.controller;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_EARLY_JSON;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.controller.v2.NQueryMetaController;
import org.junit.jupiter.api.Assertions;
@@ -38,6 +39,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+@MetadataInfo(onlyProps = true)
class NQueryMetaControllerTest {
private MockMvc mockMvc;
diff --git a/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkSourceControllerTest.java b/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkSourceControllerTest.java
index b807258079..40791a9eaa 100644
--- a/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkSourceControllerTest.java
+++ b/src/query-server/src/test/java/org/apache/kylin/rest/controller/SparkSourceControllerTest.java
@@ -21,13 +21,14 @@ package org.apache.kylin.rest.controller;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_JSON;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.DDLRequest;
import org.apache.kylin.rest.request.ExportTableRequest;
import org.apache.kylin.rest.service.SparkSourceService;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -41,6 +42,7 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+@MetadataInfo(onlyProps = true)
public class SparkSourceControllerTest {
private MockMvc mockMvc;
@@ -52,7 +54,7 @@ public class SparkSourceControllerTest {
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
- @Before
+ @BeforeEach
public void setup() {
MockitoAnnotations.initMocks(this);
mockMvc = MockMvcBuilders.standaloneSetup(sparkSourceController).defaultRequest(MockMvcRequestBuilders.get("/"))
@@ -60,7 +62,7 @@ public class SparkSourceControllerTest {
SecurityContextHolder.getContext().setAuthentication(authentication);
}
- @After
+ @AfterEach
public void tearDown() {
}
(kylin) 07/35: KYLIN-5799 Fix round, bround, lpad, rpad function
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit deb209f5319bc6d2dc0fe727d65202e3b834f55c
Author: Zhimin Wu <59...@qq.com>
AuthorDate: Wed Aug 30 18:36:05 2023 +0800
KYLIN-5799 Fix round, bround, lpad, rpad function
---
.../org/apache/kylin/query/udf/SparkMathUDF.java | 7 +++++-
.../org/apache/kylin/query/udf/SparkStringUDF.java | 15 +++++++++++--
.../kylin/query/runtime/ExpressionConverter.scala | 25 ++++------------------
.../kylin/query/runtime/SparderRexVisitor.scala | 1 -
4 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java b/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
index ae20f7d1a3..38db80697d 100644
--- a/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
+++ b/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
@@ -24,7 +24,12 @@ import org.apache.kylin.common.exception.CalciteNotSupportException;
public class SparkMathUDF implements NotConstant {
- public Object BROUND(@Parameter(name = "str1") Double exp1, @Parameter(name = "str2") Integer exp2)
+ public Object BROUND(@Parameter(name = "str1") Object exp1, @Parameter(name = "str2") Object exp2)
+ throws CalciteNotSupportException {
+ throw new CalciteNotSupportException();
+ }
+
+ public Object BROUND(@Parameter(name = "str1") Object exp1)
throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java b/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
index 867ffbefe1..93c36e6af3 100644
--- a/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
+++ b/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
@@ -64,21 +64,32 @@ public class SparkStringUDF implements NotConstant {
throw new CalciteNotSupportException();
}
- public String LPAD(@Parameter(name = "str1") String exp1, @Parameter(name = "num2") Integer exp2,
+ public String LPAD(@Parameter(name = "str1") String exp1, @Parameter(name = "num2") Object exp2,
@Parameter(name = "str3") String exp3) throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
+ public String LPAD(@Parameter(name = "str1") String exp1, @Parameter(name = "num2") Object exp2)
+ throws CalciteNotSupportException {
+ throw new CalciteNotSupportException();
+ }
+
public String REPLACE(@Parameter(name = "str1") String exp1, @Parameter(name = "str2") String exp2,
@Parameter(name = "str3") String exp3) throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
- public String RPAD(@Parameter(name = "str1") String exp1, @Parameter(name = "num2") Integer exp2,
+ public String RPAD(@Parameter(name = "str1") String exp1, @Parameter(name = "num2") Object exp2,
@Parameter(name = "str3") String exp3) throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
+ public String RPAD(@Parameter(name = "str1") String exp1, @Parameter(name = "num2") Object exp2)
+ throws CalciteNotSupportException {
+ throw new CalciteNotSupportException();
+ }
+
+
public String RTRIM(@Parameter(name = "str1") String exp1) throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
index 04afba3c94..53dc5b25a5 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
@@ -45,19 +45,20 @@ object ExpressionConverter {
"explode"
)
- private val ternaryParameterFunc = mutable.HashSet("replace", "substring_index", "lpad", "rpad", "conv", "regexp_extract")
+ private val ternaryParameterFunc = mutable.HashSet("replace", "substring_index", "conv", "regexp_extract")
private val binaryParameterFunc =
mutable.HashSet("decode", "encode", "find_in_set", "levenshtein", "sha2",
"trunc", "add_months", "date_add", "date_sub", "from_utc_timestamp", "to_utc_timestamp",
// math function
- "bround", "hypot", "log"
+ "hypot", "log"
)
private val noParameterFunc = mutable.HashSet("current_database", "input_file_block_length", "input_file_block_start",
"input_file_name", "monotonically_increasing_id", "now", "spark_partition_id", "uuid"
)
- private val varArgsFunc = mutable.HashSet("months_between", "locate", "rtrim", "from_unixtime", "to_date", "to_timestamp", "split")
+ private val varArgsFunc = mutable.HashSet("months_between", "locate", "rtrim", "from_unixtime", "to_date",
+ "to_timestamp", "split", "bround", "lpad", "rpad", "round")
private val bitmapUDF = mutable.HashSet("intersect_count_by_col", "subtract_bitmap_value", "subtract_bitmap_uuid");
@@ -205,24 +206,6 @@ object ExpressionConverter {
abs(
k_lit(children.head).cast(SparderTypeUtil
.convertSqlTypeToSparkType(relDataType)))
- case "round" =>
- var scale = children.apply(1)
- if (scale.isInstanceOf[Column]) {
- val extractConst = (scale: Column) => {
- val extractCastValue = (child: Literal) => child.value
- val reduceCast = (expr: Cast) => extractCastValue(expr.child.asInstanceOf[Literal])
- scale.expr.getClass.getSimpleName match {
- case "Cast" =>
- reduceCast(scale.expr.asInstanceOf[Cast])
- case _ =>
- throw new UnsupportedOperationException(s"Scale parameter of round function doesn't support this expression " + scale.expr.toString())
- }
- }
- scale = extractConst(scale.asInstanceOf[Column])
- }
- round(
- k_lit(children.head),
- scale.asInstanceOf[Int])
case "truncate" =>
if (children.size == 1) {
k_truncate(k_lit(children.head), 0)
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
index 2adac31699..0b7116a673 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
@@ -287,7 +287,6 @@ class SparderRexVisitor(val inputFieldNames: Seq[String],
assert(children.size == 2)
coalesce(children.apply(0).asInstanceOf[Column], k_lit(children.apply(1)))
case _ =>
-
ExpressionConverter.convert(call.getType.getSqlTypeName, call.`type`, op.getKind, op.getName, children)
}
}
(kylin) 26/35: KYLIN-5833 Use TableIndex answer select * only works for query
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 412eff5c73908508e3967412a2dbffd76d171b30
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Tue Aug 29 16:24:25 2023 +0800
KYLIN-5833 Use TableIndex answer select * only works for query
---
.../src/main/java/org/apache/kylin/common/QueryContext.java | 4 ++++
.../main/java/org/apache/kylin/query/routing/RealizationChooser.java | 4 +++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index b112591e27..b6ef63867e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -107,6 +107,10 @@ public class QueryContext implements Closeable {
@Setter
private boolean partialMatchIndex = false;
+ @Getter
+ @Setter
+ private boolean isForModeling;
+
@Getter
@Setter
private ForceToTieredStorage forcedToTieredStorage;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index 768edd91ff..56bee577c9 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -255,7 +255,9 @@ public class RealizationChooser {
buildDimensionsAndMetrics(context, dimensions, metrics);
buildStorageContext(context, dimensions, metrics, candidate);
buildSecondStorageEnabled(context.getSQLDigest());
- fixContextForTableIndexAnswerNonRawQuery(context);
+ if (!QueryContext.current().isForModeling()) {
+ fixContextForTableIndexAnswerNonRawQuery(context);
+ }
}
}
(kylin) 32/35: KYLIN-5841 Fix vulnerability, Upgrade org.apache.avro:avro to version 1.11.3
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7d24ce642c1dc96f2a476408fd546d971963d5e0
Author: huangsheng <hu...@163.com>
AuthorDate: Sun Oct 8 15:29:39 2023 +0800
KYLIN-5841 Fix vulnerability, Upgrade org.apache.avro:avro to version 1.11.3
---
pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/pom.xml b/pom.xml
index 04a01dc8a8..6b37bca758 100644
--- a/pom.xml
+++ b/pom.xml
@@ -401,6 +401,7 @@
<nimbus-jose-jwt.version>9.23</nimbus-jose-jwt.version>
<system-lambda.version>1.2.1</system-lambda.version>
<snappy-java.version>1.1.10.4</snappy-java.version>
+ <avro.version>1.11.3</avro.version>
</properties>
<modules>
@@ -3003,6 +3004,12 @@
<artifactId>netty</artifactId>
<version>${netty3.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
(kylin) 31/35: KYLIN-5840 Fix vulnerability, Upgrade org.xerial.snappy:snappy-java to version 1.1.10.4
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1c05436a01d41e490c986ee32c0ca1d07c6e90e0
Author: huangsheng <hu...@163.com>
AuthorDate: Sat Oct 7 16:46:03 2023 +0800
KYLIN-5840 Fix vulnerability, Upgrade org.xerial.snappy:snappy-java to version 1.1.10.4
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 07224bd0f1..04a01dc8a8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -400,7 +400,7 @@
<jackson-databind.version>2.13.4.2</jackson-databind.version>
<nimbus-jose-jwt.version>9.23</nimbus-jose-jwt.version>
<system-lambda.version>1.2.1</system-lambda.version>
- <snappy-java.version>1.1.10.1</snappy-java.version>
+ <snappy-java.version>1.1.10.4</snappy-java.version>
</properties>
<modules>
(kylin) 35/35: KYLIN-5846 upgrade spark version to 3.2.0-kylin-4.6.16.0
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3f9b9c83bedbce17be0dcac5af427c636353621a
Author: huangsheng <hu...@163.com>
AuthorDate: Mon Oct 30 13:30:54 2023 +0800
KYLIN-5846 upgrade spark version to 3.2.0-kylin-4.6.16.0
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 6b37bca758..889fa87c96 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@
<!-- Spark versions -->
<delta.version>2.0.2</delta.version>
- <spark.version>3.2.0-kylin-4.6.14.0</spark.version>
+ <spark.version>3.2.0-kylin-4.6.16.0</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
(kylin) 22/35: KYLIN-5828 Concurrently dict v2 jobs lead to abnormal encoding result
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c61dc4189968f29f8e56e98f72a16f100e9d6e2b
Author: huangsheng <hu...@163.com>
AuthorDate: Fri Sep 22 16:13:50 2023 +0800
KYLIN-5828 Concurrently dict v2 jobs lead to abnormal encoding result
---
.../engine/spark/builder/CreateFlatTable.scala | 15 +--
.../engine/spark/builder/DFDictionaryBuilder.scala | 16 +--
.../engine/spark/builder/DFTableEncoder.scala | 13 ++-
.../spark/builder/DictionaryBuilderHelper.java | 5 +-
.../job/stage/build/FlatTableAndDictBase.scala | 18 ++--
.../engine/spark/dict/NGlobalDictionaryV2Test.java | 77 +++++++++------
.../kylin/engine/spark/builder/TestDFChooser.scala | 5 +-
.../engine/spark/builder/TestGlobalDictBuild.scala | 58 +++++++++--
.../v3dict/GlobalDictionaryUpdateSuite.scala | 6 +-
.../scala/org/apache/spark/sql/KapFunctions.scala | 4 +-
.../sql/catalyst/expressions/KapExpresssions.scala | 24 +++--
.../org/apache/spark/sql/udf/DictEncodeImpl.scala | 9 +-
.../org/apache/spark/dict/NBucketDictionary.java | 62 +++++++++---
.../spark/dict/NGlobalDictBuilderAssist.scala | 5 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java | 10 +-
.../org/apache/spark/dict/NGlobalDictS3Store.java | 16 +--
.../org/apache/spark/dict/NGlobalDictStore.java | 4 +-
.../org/apache/spark/dict/NGlobalDictionaryV2.java | 107 +++++++++++++++++----
18 files changed, 326 insertions(+), 128 deletions(-)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
index b659b87969..654c5b9f6f 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
@@ -31,6 +31,7 @@ import org.apache.kylin.guava30.shaded.common.collect.Sets
import org.apache.kylin.metadata.cube.cuboid.NSpanningTree
import org.apache.kylin.metadata.cube.model.{NCubeJoinedFlatTableDesc, NDataSegment}
import org.apache.kylin.metadata.model._
+import org.apache.spark.dict.NGlobalDictionaryV2.NO_VERSION_SPECIFIED
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.{Dataset, Row, SparkSession}
@@ -115,12 +116,14 @@ class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
dictCols: Set[TblColRef],
encodeCols: Set[TblColRef]): Dataset[Row] = {
val ccDataset = withColumn(ds, ccCols)
+ var buildVersion = System.currentTimeMillis()
if (seg.isDictReady) {
logInfo(s"Skip already built dict, segment: ${seg.getId} of dataflow: ${seg.getDataflow.getId}")
+ buildVersion = NO_VERSION_SPECIFIED
} else {
- buildDict(ccDataset, dictCols)
+ buildDict(ccDataset, dictCols, buildVersion)
}
- encodeColumn(ccDataset, encodeCols)
+ encodeColumn(ccDataset, encodeCols, buildVersion)
}
private def withColumn(ds: Dataset[Row], withCols: Set[TblColRef]): Dataset[Row] = {
@@ -131,21 +134,21 @@ class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
withDs
}
- private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef]): Unit = {
+ private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef], buildVersion: Long): Unit = {
val matchedCols = if (seg.getIndexPlan.isSkipEncodeIntegerFamilyEnabled) {
filterOutIntegerFamilyType(ds, dictCols)
} else {
selectColumnsInTable(ds, dictCols)
}
val builder = new DFDictionaryBuilder(ds, seg, ss, Sets.newHashSet(matchedCols.asJavaCollection))
- builder.buildDictSet()
+ builder.buildDictSet(buildVersion)
}
- private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef]): Dataset[Row] = {
+ private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef], buildVersion: Long): Dataset[Row] = {
val matchedCols = selectColumnsInTable(ds, encodeCols)
var encodeDs = ds
if (matchedCols.nonEmpty) {
- encodeDs = DFTableEncoder.encodeTable(ds, seg, matchedCols.asJava)
+ encodeDs = DFTableEncoder.encodeTable(ds, seg, matchedCols.asJava, buildVersion)
}
encodeDs
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
index af8399ca3a..d930fd178b 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
@@ -43,8 +43,8 @@ class DFDictionaryBuilder(
val colRefSet: util.Set[TblColRef]) extends LogEx with Serializable {
@throws[IOException]
- def buildDictSet(): Unit = {
- colRefSet.asScala.foreach(col => safeBuild(col))
+ def buildDictSet(buildVersion: Long): Unit = {
+ colRefSet.asScala.foreach(col => safeBuild(col, buildVersion))
changeAQEConfig(true)
}
@@ -52,7 +52,7 @@ class DFDictionaryBuilder(
private val originalAQE = ss.conf.get(AQE)
@throws[IOException]
- private[builder] def safeBuild(ref: TblColRef): Unit = {
+ private[builder] def safeBuild(ref: TblColRef, buildVersion: Long): Unit = {
val sourceColumn = ref.getIdentity
ZKHelper.tryZKJaasConfiguration(ss)
val lock: Lock = KylinConfig.getInstanceFromEnv.getDistributedLockFactory
@@ -64,7 +64,7 @@ class DFDictionaryBuilder(
val bucketPartitionSize = logTime(s"calculating bucket size for $sourceColumn") {
DictionaryBuilderHelper.calculateBucketSize(seg, ref, dictColDistinct)
}
- build(ref, bucketPartitionSize, dictColDistinct)
+ build(ref, bucketPartitionSize, dictColDistinct, buildVersion)
} finally lock.unlock()
}
@@ -90,10 +90,12 @@ class DFDictionaryBuilder(
""".stripMargin
}
+
@throws[IOException]
private[builder] def build(ref: TblColRef, bucketPartitionSize: Int,
- afterDistinct: Dataset[Row]): Unit = logTime(s"building global dictionaries V2 for ${ref.getIdentity}") {
- val globalDict = new NGlobalDictionaryV2(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory)
+ afterDistinct: Dataset[Row],
+ buildVersion: Long): Unit = logTime(s"building global dictionaries V2 for ${ref.getIdentity}") {
+ val globalDict = new NGlobalDictionaryV2(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory, buildVersion)
globalDict.prepareWrite()
val broadcastDict = ss.sparkContext.broadcast(globalDict)
@@ -121,7 +123,7 @@ class DFDictionaryBuilder(
if (seg.getConfig.isGlobalDictCheckEnabled) {
logInfo(s"Start to check the correctness of the global dict, table: ${ref.getTableAlias}, col: ${ref.getName}")
- val latestGD = new NGlobalDictionaryV2(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory)
+ val latestGD = new NGlobalDictionaryV2(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory, buildVersion)
for (bid <- 0 until globalDict.getMetaInfo.getBucketSize) {
val dMap = latestGD.loadBucketDictionary(bid).getAbsoluteDictMap
val vdCount = dMap.values().stream().distinct().count()
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFTableEncoder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFTableEncoder.scala
index 9e3ca24734..a49c090838 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFTableEncoder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFTableEncoder.scala
@@ -22,6 +22,7 @@ import org.apache.kylin.engine.spark.job.NSparkCubingUtil._
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.model.TblColRef
import org.apache.spark.dict.NGlobalDictionaryV2
+import org.apache.spark.dict.NGlobalDictionaryV2.NO_VERSION_SPECIFIED
import org.apache.spark.internal.Logging
import org.apache.spark.sql.KapFunctions._
import org.apache.spark.sql.functions.{col, _}
@@ -34,7 +35,7 @@ import scala.collection.mutable._
object DFTableEncoder extends Logging {
- def encodeTable(ds: Dataset[Row], seg: NDataSegment, cols: util.Set[TblColRef]): Dataset[Row] = {
+ def encodeTable(ds: Dataset[Row], seg: NDataSegment, cols: util.Set[TblColRef], buildVersion: Long): Dataset[Row] = {
val structType = ds.schema
var partitionedDs = ds
@@ -61,17 +62,21 @@ object DFTableEncoder extends Logging {
val encodingArgs = encodingCols.map {
ref =>
- val globalDict = new NGlobalDictionaryV2(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory)
+ val globalDict = new NGlobalDictionaryV2(seg.getProject, ref.getTable,
+ ref.getName, seg.getConfig.getHdfsWorkingDirectory, buildVersion)
val bucketSize = globalDict.getBucketSizeOrDefault(seg.getConfig.getGlobalDictV2MinHashPartitions)
val enlargedBucketSize = (((minBucketSize / bucketSize) + 1) * bucketSize).toInt
-
+ logInfo(s"[EncodeTable/${buildVersion}] bucketSize:$bucketSize")
val encodeColRef = convertFromDot(ref.getBackTickIdentity)
val columnIndex = structType.fieldIndex(encodeColRef)
val dictParams = Array(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory)
.mkString(SEPARATOR)
val aliasName = structType.apply(columnIndex).name.concat(ENCODE_SUFFIX)
- val encodeCol = dict_encode(col(encodeColRef).cast(StringType), lit(dictParams), lit(bucketSize).cast(StringType)).as(aliasName)
+ val encodeCol = dict_encode(col(encodeColRef).cast(StringType),
+ lit(dictParams),
+ lit(bucketSize).cast(StringType),
+ lit(buildVersion).cast(LongType)).as(aliasName)
val columns = encodeCol
(enlargedBucketSize, col(encodeColRef).cast(StringType), columns, aliasName,
bucketSize == 1)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java
index df0288164d..97ef4f433c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DictionaryBuilderHelper.java
@@ -58,8 +58,9 @@ public class DictionaryBuilderHelper {
*/
public static int calculateBucketSize(NDataSegment seg, TblColRef col, Dataset<Row> afterDistinct)
throws IOException {
+ long resizeVersion = System.currentTimeMillis();
NGlobalDictionaryV2 globalDict = new NGlobalDictionaryV2(seg.getProject(), col.getTable(), col.getName(),
- seg.getConfig().getHdfsWorkingDirectory());
+ seg.getConfig().getHdfsWorkingDirectory(), resizeVersion);
int bucketPartitionSize = globalDict.getBucketSizeOrDefault(seg.getConfig().getGlobalDictV2MinHashPartitions());
int bucketThreshold = seg.getConfig().getGlobalDictV2ThresholdBucketSize();
int resizeBucketSize = bucketPartitionSize;
@@ -107,7 +108,7 @@ public class DictionaryBuilderHelper {
if (resizeBucketSize != bucketPartitionSize) {
logger.info("Start building a global dictionary column for {}, need resize from {} to {} ",
col.getName(), bucketPartitionSize, resizeBucketSize);
- resize(col, seg, resizeBucketSize, afterDistinct.sparkSession());
+ resize(col, seg, resizeBucketSize, afterDistinct.sparkSession(), resizeVersion);
logger.info("End building a global dictionary column for {}, need resize from {} to {} ", col.getName(),
bucketPartitionSize, resizeBucketSize);
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 935eaefaea..bacd24c225 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -44,6 +44,7 @@ import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBu
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.cube.planner.CostBasePlannerUtils
import org.apache.kylin.metadata.model._
+import org.apache.spark.dict.NGlobalDictionaryV2.NO_VERSION_SPECIFIED
import org.apache.spark.sql.KapFunctions.dict_encode_v3
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.plans.JoinType
@@ -563,18 +564,20 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
if (dictCols.isEmpty && encodeCols.isEmpty) {
return table
}
+ var buildVersion = System.currentTimeMillis()
if (dataSegment.isDictReady) {
logInfo(s"Skip DICTIONARY segment $segmentId")
+ buildVersion = NO_VERSION_SPECIFIED
} else {
// ensure at least one worker was registered before dictionary lock added.
waitTillWorkerRegistered()
- buildDict(table, dictCols)
+ buildDict(table, dictCols, buildVersion)
}
if (config.isV3DictEnable) {
buildV3DictIfNeeded(table, encodeCols)
} else {
- encodeColumn(table, encodeCols)
+ encodeColumn(table, encodeCols, buildVersion)
}
}
@@ -632,23 +635,24 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
tableWithCcs
}
- private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef]): Unit = {
+ private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef], buildVersion: Long): Unit = {
if (config.isV2DictEnable) {
- logInfo("Build v2 dict default.")
+ logInfo(s"Build v2 dict default. " +
+ s"[${dataSegment.getModel.getAlias}] model dict build version is $buildVersion")
var matchedCols = selectColumnsInTable(ds, dictCols)
if (dataSegment.getIndexPlan.isSkipEncodeIntegerFamilyEnabled) {
matchedCols = matchedCols.filterNot(_.getType.isIntegerFamily)
}
val builder = new DFDictionaryBuilder(ds, dataSegment, sparkSession, Sets.newHashSet(matchedCols.asJavaCollection))
- builder.buildDictSet()
+ builder.buildDictSet(buildVersion)
}
}
- private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef]): Dataset[Row] = {
+ private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef], buildVersion: Long): Dataset[Row] = {
val matchedCols = selectColumnsInTable(ds, encodeCols)
var encodeDs = ds
if (matchedCols.nonEmpty) {
- encodeDs = DFTableEncoder.encodeTable(ds, dataSegment, matchedCols.asJava)
+ encodeDs = DFTableEncoder.encodeTable(ds, dataSegment, matchedCols.asJava, buildVersion)
}
encodeDs
}
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/dict/NGlobalDictionaryV2Test.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/dict/NGlobalDictionaryV2Test.java
index 8d29579e4a..6f30eb2fc3 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/dict/NGlobalDictionaryV2Test.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/dict/NGlobalDictionaryV2Test.java
@@ -35,6 +35,7 @@ import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.dict.NBucketDictionary;
import org.apache.spark.dict.NGlobalDictHDFSStore;
import org.apache.spark.dict.NGlobalDictMetaInfo;
+import org.apache.spark.dict.NGlobalDictS3Store;
import org.apache.spark.dict.NGlobalDictStore;
import org.apache.spark.dict.NGlobalDictionaryV2;
import org.apache.spark.sql.Dataset;
@@ -70,33 +71,33 @@ public class NGlobalDictionaryV2Test extends NLocalWithSparkSessionTest {
@Test
public void testGlobalDictHDFSStoreRoundTest() throws IOException {
- testAll();
+ testAll(false);
}
@Test
public void testGlobalDictS3StoreRoundTest() throws IOException {
// global s3 dict store
overwriteSystemProp("kylin.engine.global-dict.store.impl", "org.apache.spark.dict.NGlobalDictS3Store");
- testAll();
+ testAll(true);
}
- private void testAll() throws IOException {
- roundTest(5);
- roundTest(50);
- roundTest(500);
+ private void testAll(boolean isS3Store) throws IOException {
+ roundTest(5, isS3Store);
+ roundTest(50, isS3Store);
+ roundTest(500, isS3Store);
}
- private void roundTest(int size) throws IOException {
+ private void roundTest(int size, boolean isS3Store) throws IOException {
System.out.println("NGlobalDictionaryV2Test -> roundTest -> " + System.currentTimeMillis());
KylinConfig config = KylinConfig.getInstanceFromEnv();
- NGlobalDictionaryV2 dict1 = new NGlobalDictionaryV2("t1", "a", "spark", config.getHdfsWorkingDirectory());
- NGlobalDictionaryV2 dict2 = new NGlobalDictionaryV2("t2", "a", "local", config.getHdfsWorkingDirectory());
+ NGlobalDictionaryV2 dict1 = new NGlobalDictionaryV2("t1", "a", "spark", config.getHdfsWorkingDirectory(), System.currentTimeMillis());
+ NGlobalDictionaryV2 dict2 = new NGlobalDictionaryV2("t2", "a", "local", config.getHdfsWorkingDirectory(), System.currentTimeMillis());
List<String> stringList = generateRandomData(size);
Collections.sort(stringList);
- runWithSparkBuildGlobalDict(dict1, stringList);
- runWithLocalBuildGlobalDict(dict2, stringList);
- compareTwoVersionDict(dict1, dict2);
- compareTwoModeVersionNum(dict1, dict2);
+ runWithSparkBuildGlobalDict(dict1, stringList, isS3Store);
+ runWithLocalBuildGlobalDict(dict2, stringList, isS3Store);
+ compareTwoVersionDict(dict1, dict2, isS3Store);
+ compareTwoModeVersionNum(dict1, dict2, isS3Store);
}
private List<String> generateRandomData(int size) {
@@ -107,9 +108,14 @@ public class NGlobalDictionaryV2Test extends NLocalWithSparkSessionTest {
return stringList;
}
- private void runWithSparkBuildGlobalDict(NGlobalDictionaryV2 dict, List<String> stringSet) throws IOException {
+ private void runWithSparkBuildGlobalDict(NGlobalDictionaryV2 dict, List<String> stringSet, boolean isS3Store) throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- dict.prepareWrite();
+ if (isS3Store) {
+ dict.prepareWriteS3();
+ } else {
+ dict.prepareWrite();
+ }
+
List<Row> rowList = Lists.newLinkedList();
for (String str : stringSet) {
rowList.add(RowFactory.create(str));
@@ -122,7 +128,7 @@ public class NGlobalDictionaryV2Test extends NLocalWithSparkSessionTest {
return new Tuple2<>(row.get(0).toString(), null);
}).sortByKey().partitionBy(new HashPartitioner(BUCKET_SIZE)).mapPartitionsWithIndex(
(Function2<Integer, Iterator<Tuple2<String, String>>, Iterator<Object>>) (bucketId, tuple2Iterator) -> {
- NBucketDictionary bucketDict = dict.loadBucketDictionary(bucketId);
+ NBucketDictionary bucketDict = isS3Store? dict.loadBucketDictionaryForTestS3(bucketId) : dict.loadBucketDictionary(bucketId);
while (tuple2Iterator.hasNext()) {
Tuple2<String, String> tuple2 = tuple2Iterator.next();
bucketDict.addRelativeValue(tuple2._1);
@@ -131,12 +137,20 @@ public class NGlobalDictionaryV2Test extends NLocalWithSparkSessionTest {
return Collections.emptyIterator();
}, true).count();
- dict.writeMetaDict(BUCKET_SIZE, config.getGlobalDictV2MaxVersions(), config.getGlobalDictV2VersionTTL());
+ if (isS3Store) {
+ dict.writeMetaDictForTestS3(BUCKET_SIZE, config.getGlobalDictV2MaxVersions(), config.getGlobalDictV2VersionTTL());
+ } else {
+ dict.writeMetaDict(BUCKET_SIZE, config.getGlobalDictV2MaxVersions(), config.getGlobalDictV2VersionTTL());
+ }
}
- private void runWithLocalBuildGlobalDict(NGlobalDictionaryV2 dict, List<String> stringSet) throws IOException {
+ private void runWithLocalBuildGlobalDict(NGlobalDictionaryV2 dict, List<String> stringSet, boolean isS3Store) throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- dict.prepareWrite();
+ if (isS3Store) {
+ dict.prepareWriteS3();
+ } else {
+ dict.prepareWrite();
+ }
HashPartitioner partitioner = new HashPartitioner(BUCKET_SIZE);
Map<Integer, List<String>> vmap = new HashMap<>();
for (int i = 0; i < BUCKET_SIZE; i++) {
@@ -148,31 +162,36 @@ public class NGlobalDictionaryV2Test extends NLocalWithSparkSessionTest {
}
for (Map.Entry<Integer, List<String>> entry : vmap.entrySet()) {
- NBucketDictionary bucketDict = dict.loadBucketDictionary(entry.getKey());
+ NBucketDictionary bucketDict = isS3Store ? dict.loadBucketDictionaryForTestS3(entry.getKey()):dict.loadBucketDictionary(entry.getKey());
for (String s : entry.getValue()) {
bucketDict.addRelativeValue(s);
}
bucketDict.saveBucketDict(entry.getKey());
}
- dict.writeMetaDict(BUCKET_SIZE, config.getGlobalDictV2MaxVersions(), config.getGlobalDictV2VersionTTL());
+ if (isS3Store) {
+ dict.writeMetaDictForTestS3(BUCKET_SIZE, config.getGlobalDictV2MaxVersions(), config.getGlobalDictV2VersionTTL());
+ } else {
+ dict.writeMetaDict(BUCKET_SIZE, config.getGlobalDictV2MaxVersions(), config.getGlobalDictV2VersionTTL());
+ }
+
}
- private void compareTwoModeVersionNum(NGlobalDictionaryV2 dict1, NGlobalDictionaryV2 dict2) throws IOException {
- NGlobalDictStore store1 = new NGlobalDictHDFSStore(dict1.getResourceDir());
- NGlobalDictStore store2 = new NGlobalDictHDFSStore(dict2.getResourceDir());
+ private void compareTwoModeVersionNum(NGlobalDictionaryV2 dict1, NGlobalDictionaryV2 dict2, boolean isS3Store) throws IOException {
+ NGlobalDictStore store1 = isS3Store? new NGlobalDictS3Store(dict1.getResourceDir()) : new NGlobalDictHDFSStore(dict1.getResourceDir());
+ NGlobalDictStore store2 = isS3Store? new NGlobalDictS3Store(dict2.getResourceDir()) : new NGlobalDictHDFSStore(dict2.getResourceDir());
Assert.assertEquals(store1.listAllVersions().length, store2.listAllVersions().length);
}
- private void compareTwoVersionDict(NGlobalDictionaryV2 dict1, NGlobalDictionaryV2 dict2) throws IOException {
- NGlobalDictMetaInfo metadata1 = dict1.getMetaInfo();
- NGlobalDictMetaInfo metadata2 = dict2.getMetaInfo();
+ private void compareTwoVersionDict(NGlobalDictionaryV2 dict1, NGlobalDictionaryV2 dict2, boolean isS3Store) throws IOException {
+ NGlobalDictMetaInfo metadata1 = isS3Store? dict1.getMetaInfoForTestS3():dict1.getMetaInfo();
+ NGlobalDictMetaInfo metadata2 = isS3Store? dict2.getMetaInfoForTestS3():dict2.getMetaInfo();
// compare dict meta info
Assert.assertEquals(metadata1, metadata2);
for (int i = 0; i < metadata1.getBucketSize(); i++) {
- NBucketDictionary bucket1 = dict1.loadBucketDictionary(i);
- NBucketDictionary bucket2 = dict2.loadBucketDictionary(i);
+ NBucketDictionary bucket1 = isS3Store? dict1.loadBucketDictionaryForTestS3(i):dict1.loadBucketDictionary(i);
+ NBucketDictionary bucket2 = isS3Store? dict2.loadBucketDictionaryForTestS3(i):dict2.loadBucketDictionary(i);
Object2LongMap<String> map1 = bucket1.getAbsoluteDictMap();
Object2LongMap<String> map2 = bucket2.getAbsoluteDictMap();
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDFChooser.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDFChooser.scala
index 9dd526d6bd..5ebedddd13 100644
--- a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDFChooser.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestDFChooser.scala
@@ -160,8 +160,9 @@ class TestDFChooser extends SparderBaseFunSuite with SharedSparkSession with Loc
val dict1 = new NGlobalDictionaryV2(seg.getProject, col.getTable, col.getName, seg.getConfig.getHdfsWorkingDirectory)
val meta1 = dict1.getMetaInfo
val needResizeBucketSize = dict1.getBucketSizeOrDefault(seg.getConfig.getGlobalDictV2MinHashPartitions) + 10
- NGlobalDictBuilderAssist.resize(col, seg, needResizeBucketSize, spark)
- val dict2 = new NGlobalDictionaryV2(seg.getProject, col.getTable, col.getName, seg.getConfig.getHdfsWorkingDirectory)
+ val resizeVersion = System.currentTimeMillis()
+ NGlobalDictBuilderAssist.resize(col, seg, needResizeBucketSize, spark, resizeVersion)
+ val dict2 = new NGlobalDictionaryV2(seg.getProject, col.getTable, col.getName, seg.getConfig.getHdfsWorkingDirectory, resizeVersion)
Assert.assertEquals(meta1.getDictCount, dict2.getMetaInfo.getDictCount)
Assert.assertEquals(meta1.getBucketSize + 10, dict2.getMetaInfo.getBucketSize)
}
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestGlobalDictBuild.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestGlobalDictBuild.scala
index 764229ff44..016844e7c8 100644
--- a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestGlobalDictBuild.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/TestGlobalDictBuild.scala
@@ -17,10 +17,12 @@
*/
package org.apache.kylin.engine.spark.builder
-import org.apache.commons.lang3.RandomStringUtils
+import org.apache.commons.lang3.{RandomStringUtils, StringUtils}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{Path, PathFilter}
import org.apache.kylin.common.KylinConfig
+import org.apache.kylin.engine.spark.builder.v3dict.GlobalDictionaryBuilderHelper.genRandomData
+import org.apache.kylin.engine.spark.job.NSparkCubingUtil
import org.apache.kylin.metadata.cube.cuboid.{AdaptiveSpanningTree, NSpanningTreeFactory}
import org.apache.kylin.metadata.cube.model.{NDataSegment, NDataflow, NDataflowManager}
import org.apache.kylin.metadata.model.TblColRef
@@ -48,6 +50,40 @@ class TestGlobalDictBuild extends SparderBaseFunSuite with SharedSparkSession wi
config
}
+ test("build v2 dict and encode flattable") {
+ val dsMgr: NDataflowManager = NDataflowManager.getInstance(getTestConfig, DEFAULT_PROJECT)
+ val df: NDataflow = dsMgr.getDataflow(CUBE_NAME)
+ val seg = df.getLastSegment
+ val nSpanningTree = NSpanningTreeFactory.fromLayouts(seg.getIndexPlan.getAllLayouts, df.getUuid)
+ val dictColSet = DictionaryBuilderHelper.extractTreeRelatedGlobalDicts(seg, nSpanningTree.getAllIndexEntities)
+
+ val dictCol = dictColSet.iterator().next()
+ val encodeColName: String = StringUtils.split(dictCol.getTable, ".").apply(1) + NSparkCubingUtil.SEPARATOR + dictCol.getName
+ val randomDF = genRandomData(spark, encodeColName, 1000, 10)
+
+ val dictionaryBuilder = new DFDictionaryBuilder(randomDF, seg, randomDF.sparkSession, dictColSet)
+ val colName = dictColSet.iterator().next()
+ val bucketPartitionSize = DictionaryBuilderHelper.calculateBucketSize(seg, colName, randomDF)
+ val buildVersion = System.currentTimeMillis()
+ dictionaryBuilder.build(colName, bucketPartitionSize, randomDF, buildVersion)
+ val dict = new NGlobalDictionaryV2(seg.getProject,
+ colName.getTable,
+ colName.getName,
+ seg.getConfig.getHdfsWorkingDirectory,
+ buildVersion)
+ val meta1 = dict.getMetaInfo
+ Assert.assertEquals(1000, meta1.getDictCount)
+ val encode = encodeColumn(randomDF, seg, dictColSet, buildVersion)
+ val rowsWithZero = encode.filter(encode(encode.columns(1)) === 0)
+ Assert.assertEquals(true, rowsWithZero.isEmpty)
+ // clean all
+ val cleanCol = dictColSet.iterator().next()
+ val cleanDict = new NGlobalDictionaryV2(seg.getProject, cleanCol.getTable, cleanCol.getName, seg.getConfig.getHdfsWorkingDirectory)
+ val cleanDictPath = new Path(seg.getConfig.getHdfsWorkingDirectory + cleanDict.getResourceDir)
+ val fileSystem = cleanDictPath.getFileSystem(new Configuration())
+ fileSystem.delete(cleanDictPath, true)
+ }
+
test("global dict build and checkout bucket resize strategy") {
val dsMgr: NDataflowManager = NDataflowManager.getInstance(getTestConfig, DEFAULT_PROJECT)
Assert.assertTrue(getTestConfig.getHdfsWorkingDirectory.startsWith("file:"))
@@ -150,10 +186,11 @@ class TestGlobalDictBuild extends SparderBaseFunSuite with SharedSparkSession wi
val dictColumn = col(dataflow.getModel.getColumnIdByColumnName(dictCol.getIdentity).toString)
val distinctDS = sampleDS.select(dictColumn).distinct()
val dictBuilder = new DFDictionaryBuilder(sampleDS, segment, spark, dictColSet)
- dictBuilder.build(dictCol, shufflePartitionSizeInt, distinctDS)
+ val buildVersion = System.currentTimeMillis()
+ dictBuilder.build(dictCol, shufflePartitionSizeInt, distinctDS, buildVersion)
val dictStore = NGlobalDictStoreFactory.getResourceStore(segment.getConfig.getHdfsWorkingDirectory + dictV2.getResourceDir)
- val versionPath = dictStore.getVersionDir(dictStore.listAllVersions()(0))
+ val versionPath = dictStore.getVersionDir(buildVersion)
val dictDirSize = fileSystem.listStatus(versionPath, new PathFilter {
override def accept(path: Path): Boolean = {
@@ -184,7 +221,7 @@ class TestGlobalDictBuild extends SparderBaseFunSuite with SharedSparkSession wi
seg.getConfig.setProperty("kylin.engine.global-dict-aqe-enabled", "FALSE")
dictionaryBuilder.changeAQEConfig(false)
Assert.assertFalse(spark.conf.get("spark.sql.adaptive.enabled").toBoolean)
- dictionaryBuilder.build(col, bucketPartitionSize, ds)
+ dictionaryBuilder.build(col, bucketPartitionSize, ds, System.currentTimeMillis())
Assert.assertTrue(spark.conf.get("spark.sql.adaptive.enabled").equals(originalAQE))
// false true
@@ -196,7 +233,7 @@ class TestGlobalDictBuild extends SparderBaseFunSuite with SharedSparkSession wi
dictionaryBuilder.changeAQEConfig(false)
Assert.assertTrue(spark.conf.get("spark.sql.adaptive.enabled").equals(originalAQE))
- dictionaryBuilder.build(col, bucketPartitionSize, ds)
+ dictionaryBuilder.build(col, bucketPartitionSize, ds, System.currentTimeMillis())
Assert.assertTrue(spark.conf.get("spark.sql.adaptive.enabled").equals(originalAQE))
// true true
@@ -210,12 +247,19 @@ class TestGlobalDictBuild extends SparderBaseFunSuite with SharedSparkSession wi
val col = dictColSet.iterator().next()
val ds = randomDataSet.select("26").distinct()
val bucketPartitionSize = DictionaryBuilderHelper.calculateBucketSize(seg, col, ds)
- dictionaryBuilder.build(col, bucketPartitionSize, ds)
+ val buildVersion = System.currentTimeMillis()
+ dictionaryBuilder.build(col, bucketPartitionSize, ds, buildVersion)
val dict = new NGlobalDictionaryV2(seg.getProject, col.getTable, col.getName,
- seg.getConfig.getHdfsWorkingDirectory)
+ seg.getConfig.getHdfsWorkingDirectory, buildVersion)
dict.getMetaInfo
}
+ def encodeColumn(ds: Dataset[Row], dataSegment: NDataSegment,
+ encodeCols: Set[TblColRef], buildVersion: Long): Dataset[Row] = {
+ val encodeDs = DFTableEncoder.encodeTable(ds, dataSegment, encodeCols, buildVersion)
+ encodeDs
+ }
+
def generateOriginData(count: Int, length: Int): Dataset[Row] = {
var schema = new StructType
diff --git a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/v3dict/GlobalDictionaryUpdateSuite.scala b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/v3dict/GlobalDictionaryUpdateSuite.scala
index 71b49a0924..e089abcdb8 100644
--- a/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/v3dict/GlobalDictionaryUpdateSuite.scala
+++ b/src/spark-project/engine-spark/src/test/scala/org/apache/kylin/engine/spark/builder/v3dict/GlobalDictionaryUpdateSuite.scala
@@ -87,11 +87,13 @@ class GlobalDictionaryUpdateSuite extends SparderBaseFunSuite with LocalMetadata
val dictionaryBuilder = new DFDictionaryBuilder(randomDataSet, seg, randomDataSet.sparkSession, dictColSet)
val colName = dictColSet.iterator().next()
val bucketPartitionSize = DictionaryBuilderHelper.calculateBucketSize(seg, colName, randomDataSet)
- dictionaryBuilder.build(colName, bucketPartitionSize, randomDataSet)
+ val buildVersion = System.currentTimeMillis()
+ dictionaryBuilder.build(colName, bucketPartitionSize, randomDataSet, buildVersion)
val dict = new NGlobalDictionaryV2(seg.getProject,
colName.getTable,
colName.getName,
- seg.getConfig.getHdfsWorkingDirectory)
+ seg.getConfig.getHdfsWorkingDirectory,
+ buildVersion)
dict.getMetaInfo
}
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala
index 77cce256e1..bf18b4aee7 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala
@@ -247,8 +247,8 @@ object KapFunctions {
}
}
- def dict_encode(column: Column, dictParams: Column, bucketSize: Column): Column = {
- Column(DictEncode(column.expr, dictParams.expr, bucketSize.expr))
+ def dict_encode(column: Column, dictParams: Column, bucketSize: Column, buildVersion: Column): Column = {
+ Column(DictEncode(column.expr, dictParams.expr, bucketSize.expr, buildVersion.expr))
}
def dict_encode_v3(column: Column, colName: String): Column = {
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
index 74beffaae3..a27fff2ad1 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
@@ -366,7 +366,9 @@ case class DictEncodeV3(child: Expression, col: String) extends UnaryExpression
override protected def nullSafeEval(input: Any): Any = super.nullSafeEval(input)
}
-case class DictEncode(left: Expression, mid: Expression, right: Expression) extends TernaryExpression with ExpectsInputTypes {
+case class DictEncode(left: Expression, mid: Expression,
+ right: Expression, buildVersion: Expression)
+ extends QuaternaryExpression with ExpectsInputTypes {
def maxFields: Int = SQLConf.get.maxToStringFields
@@ -376,7 +378,9 @@ case class DictEncode(left: Expression, mid: Expression, right: Expression) exte
override def third: Expression = right
- override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType, StringType, StringType)
+ override def fourth: Expression = buildVersion
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType, StringType, StringType, LongType)
override protected def doGenCode(ctx: CodegenContext,
ev: ExprCode): ExprCode = {
@@ -395,13 +399,14 @@ case class DictEncode(left: Expression, mid: Expression, right: Expression) exte
val dictParamsTerm = mid.simpleString(maxFields)
val bucketSizeTerm = right.simpleString(maxFields).toInt
+ val version = buildVersion.simpleString(maxFields).toLong
val initBucketDictFuncName = ctx.addNewFunction(s"init${bucketDictTerm.replace("[", "").replace("]", "")}BucketDict",
s"""
| private void init${bucketDictTerm.replace("[", "").replace("]", "")}BucketDict(int idx) {
| try {
| int bucketId = idx % $bucketSizeTerm;
- | $globalDictTerm = new org.apache.spark.dict.NGlobalDictionaryV2("$dictParamsTerm");
+ | $globalDictTerm = new org.apache.spark.dict.NGlobalDictionaryV2("$dictParamsTerm",${version}L);
| $bucketDictTerm = $globalDictTerm.loadBucketDictionary(bucketId, true);
| } catch (Exception e) {
| throw new RuntimeException(e);
@@ -411,13 +416,13 @@ case class DictEncode(left: Expression, mid: Expression, right: Expression) exte
ctx.addPartitionInitializationStatement(s"$initBucketDictFuncName(partitionIndex);");
- defineCodeGen(ctx, ev, (arg1, arg2, arg3) => {
+ defineCodeGen(ctx, ev, (arg1, arg2, arg3, arg4) => {
s"""$bucketDictTerm.encode($arg1)"""
})
}
- override protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
- DictEncodeImpl.evaluate(input1.toString, input2.toString, input3.toString)
+ override protected def nullSafeEval(input1: Any, input2: Any, input3: Any, input4: Any): Any = {
+ DictEncodeImpl.evaluate(input1.toString, input2.toString, input3.toString, input4.toString)
}
override def eval(input: InternalRow): Any = {
@@ -432,8 +437,11 @@ case class DictEncode(left: Expression, mid: Expression, right: Expression) exte
override def prettyName: String = "DICTENCODE"
- override protected def withNewChildrenInternal(newFirst: Expression, newSecond: Expression, newThird: Expression): Expression = {
- val newChildren = Seq(newFirst, newSecond, newThird)
+ override protected def withNewChildrenInternal(newFirst: Expression,
+ newSecond: Expression,
+ newThird: Expression,
+ newFourth: Expression): Expression = {
+ val newChildren = Seq(newFirst, newSecond, newThird, newFourth)
super.legacyWithNewChildren(newChildren)
}
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
index a33f29d736..afd1d8fbcd 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
@@ -30,19 +30,18 @@ object DictEncodeImpl {
override def initialValue(): util.HashMap[String, NBucketDictionary] = new util.HashMap[String, NBucketDictionary]()
}
- def evaluate(inputValue: String, dictParams: String, bucketSize: String): Long = {
+ def evaluate(inputValue: String, dictParams: String, bucketSize: String, buildVersion: String): Long = {
var cachedBucketDict = DictEncodeImpl.cacheBucketDict.get().get(dictParams)
if (cachedBucketDict == null) {
- cachedBucketDict = initBucketDict(dictParams, bucketSize)
+ cachedBucketDict = initBucketDict(dictParams, bucketSize, buildVersion)
}
cachedBucketDict.encode(inputValue)
}
- private def initBucketDict(dictParams: String, bucketSize: String): NBucketDictionary = {
+ private def initBucketDict(dictParams: String, bucketSize: String, buildVersion: String): NBucketDictionary = {
val partitionID = TaskContext.get.partitionId
val encodeBucketId = partitionID % bucketSize.toInt
- val globalDict = new NGlobalDictionaryV2(dictParams)
-
+ val globalDict = new NGlobalDictionaryV2(dictParams, buildVersion.toLong)
val cachedBucketDict = globalDict.loadBucketDictionary(encodeBucketId, true)
DictEncodeImpl.cacheBucketDict.get.put(dictParams, cachedBucketDict)
TaskContext.get().addTaskCompletionListener(new TaskCompletionListener {
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
index e7fb062c1d..024b88c907 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
@@ -18,6 +18,9 @@
package org.apache.spark.dict;
import java.io.IOException;
+import java.util.Arrays;
+
+import com.alibaba.nacos.common.JustForTest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,6 +28,8 @@ import org.slf4j.LoggerFactory;
import it.unimi.dsi.fastutil.objects.Object2LongMap;
import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
+import static org.apache.spark.dict.NGlobalDictionaryV2.NO_VERSION_SPECIFIED;
+
public class NBucketDictionary {
protected static final Logger logger = LoggerFactory.getLogger(NBucketDictionary.class);
@@ -33,30 +38,58 @@ public class NBucketDictionary {
private int bucketId;
+ private long buildVersion;
+
private Object2LongMap<String> absoluteDictMap;
// Relative dictionary needs to calculate dictionary code according to NGlobalDictMetaInfo's bucketOffsets
private Object2LongMap<String> relativeDictMap;
- NBucketDictionary(String baseDir, String workingDir, int bucketId, NGlobalDictMetaInfo metainfo)
- throws IOException {
- this(baseDir, workingDir, bucketId, metainfo, false);
- }
-
- NBucketDictionary(String baseDir, String workingDir, int bucketId, NGlobalDictMetaInfo metainfo, boolean isForColumnEncoding)
- throws IOException {
- this.workingDir = workingDir;
- this.bucketId = bucketId;
- final NGlobalDictStore globalDictStore = NGlobalDictStoreFactory.getResourceStore(baseDir);
+ private void initDictMap(int bucketId, NGlobalDictMetaInfo metainfo,
+ boolean isForColumnEncoding, long buildVersion, NGlobalDictStore globalDictStore) throws IOException {
Long[] versions = globalDictStore.listAllVersions();
logger.debug("versions.length is {}", versions.length);
if (versions.length == 0) {
this.absoluteDictMap = new Object2LongOpenHashMap<>();
- } else {
+ } else if (buildVersion == NO_VERSION_SPECIFIED || !Arrays.asList(versions).contains(buildVersion)) {
+ logger.info("Initializes dict map with the latest version:{}",versions[versions.length - 1]);
this.absoluteDictMap = globalDictStore.getBucketDict(versions[versions.length - 1], metainfo, bucketId, isForColumnEncoding);
+ } else {
+ logger.info("Initializes dict map with the specified version:{}",buildVersion);
+ this.absoluteDictMap = globalDictStore.getBucketDict(buildVersion, metainfo, bucketId, isForColumnEncoding);
}
this.relativeDictMap = new Object2LongOpenHashMap<>();
}
+ private void initDictMap(String baseDir, int bucketId, NGlobalDictMetaInfo metainfo, boolean isForColumnEncoding,
+ long buildVersion) throws IOException {
+ final NGlobalDictStore globalDictStore = NGlobalDictStoreFactory.getResourceStore(baseDir);
+ initDictMap(bucketId, metainfo, isForColumnEncoding, buildVersion, globalDictStore);
+ }
+
+ @JustForTest
+ private void initDictMapForS3Test(String baseDir, int bucketId, NGlobalDictMetaInfo metainfo, boolean isForColumnEncoding, long buildVersion) throws IOException {
+ final NGlobalDictStore globalDictStore = new NGlobalDictS3Store(baseDir);
+ initDictMap(bucketId, metainfo, isForColumnEncoding, buildVersion, globalDictStore);
+ }
+
+ @JustForTest
+ NBucketDictionary(String baseDir, String workingDir, int bucketId, NGlobalDictMetaInfo metainfo
+ , boolean isForColumnEncoding, long buildVersion, boolean justForTest)
+ throws IOException {
+ this.workingDir = workingDir;
+ this.bucketId = bucketId;
+ this.buildVersion = buildVersion;
+ initDictMapForS3Test(baseDir, bucketId, metainfo, isForColumnEncoding, this.buildVersion);
+ }
+
+ NBucketDictionary(String baseDir, String workingDir, int bucketId, NGlobalDictMetaInfo metainfo, boolean isForColumnEncoding, long buildVersion)
+ throws IOException {
+ this.workingDir = workingDir;
+ this.bucketId = bucketId;
+ this.buildVersion = buildVersion;
+ initDictMap(baseDir, bucketId, metainfo, isForColumnEncoding, this.buildVersion);
+ }
+
NBucketDictionary(String workingDir) {
this.workingDir = workingDir;
this.absoluteDictMap = new Object2LongOpenHashMap<>();
@@ -78,7 +111,12 @@ public class NBucketDictionary {
}
public long encode(Object value) {
- return absoluteDictMap.getLong(value.toString());
+ long encodeValue = absoluteDictMap.getLong(value.toString());
+ if (encodeValue == 0){
+ throw new IllegalArgumentException(String.format("DFTable encode key:%s with error value:%s",
+ value.toString(), encodeValue ));
+ }
+ return encodeValue;
}
public void saveBucketDict(int bucketId) throws IOException {
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictBuilderAssist.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictBuilderAssist.scala
index 41c4389d52..76b39cd67d 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictBuilderAssist.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictBuilderAssist.scala
@@ -34,8 +34,9 @@ import scala.collection.JavaConverters._
object NGlobalDictBuilderAssist extends Logging {
@throws[IOException]
- def resize(ref: TblColRef, seg: NDataSegment, bucketPartitionSize: Int, ss: SparkSession): Unit = {
- val globalDict = new NGlobalDictionaryV2(seg.getProject, ref.getTable, ref.getName, seg.getConfig.getHdfsWorkingDirectory)
+ def resize(ref: TblColRef, seg: NDataSegment, bucketPartitionSize: Int, ss: SparkSession, resizeVersion: Long): Unit = {
+ val globalDict = new NGlobalDictionaryV2(seg.getProject, ref.getTable,
+ ref.getName, seg.getConfig.getHdfsWorkingDirectory, resizeVersion)
val broadcastDict = ss.sparkContext.broadcast(globalDict)
globalDict.prepareWrite()
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java
index 6aa9ade586..49c6a30ec1 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictHDFSStore.java
@@ -38,6 +38,8 @@ import org.slf4j.LoggerFactory;
import it.unimi.dsi.fastutil.objects.Object2LongMap;
import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
+import static org.apache.spark.dict.NGlobalDictionaryV2.NO_VERSION_SPECIFIED;
+
public class NGlobalDictHDFSStore implements NGlobalDictStore {
protected static final String VERSION_PREFIX = "version_";
@@ -381,17 +383,17 @@ public class NGlobalDictHDFSStore implements NGlobalDictStore {
}
@Override
- public void commit(String workingDir, int maxVersions, long versionTTL) throws IOException {
+ public void commit(String workingDir, int maxVersions, long versionTTL, long buildVersion) throws IOException {
Path workingPath = new Path(workingDir);
- // copy working dir to newVersion dir
- Path newVersionPath = new Path(basePath, VERSION_PREFIX + System.currentTimeMillis());
+ final long commitVersion = buildVersion == NO_VERSION_SPECIFIED ? System.currentTimeMillis() : buildVersion;
+ Path newVersionPath = new Path(basePath, VERSION_PREFIX + commitVersion);
fileSystem.rename(workingPath, newVersionPath);
logger.info("Commit from {} to {}", workingPath, newVersionPath);
cleanUp(maxVersions, versionTTL);
}
@Override
- public String getWorkingDir() {
+ public String getWorkingDir(long buildVersion) {
return baseDir + WORKING_DIR;
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictS3Store.java b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictS3Store.java
index 01b42b9699..24e6d046e6 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictS3Store.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictS3Store.java
@@ -71,21 +71,25 @@ public class NGlobalDictS3Store extends NGlobalDictHDFSStore {
}
@Override
- public void commit(String workingDir, int maxVersions, long versionTTL) throws IOException {
+ public void commit(String workingDir, int maxVersions, long versionTTL, long buildVersion) throws IOException {
cleanWorkingFlagPath(basePath);
logger.info("Commit {}", workingDir);
cleanUp(maxVersions, versionTTL);
}
@Override
- public String getWorkingDir() {
- Path path = getWorkingFlagPath(basePath);
+ public String getWorkingDir(long buildVersion) {
+ Path path = getWorkingFlagPath(basePath, buildVersion);
long timestamp = Long.parseLong(path.getName().substring(WORKING_PREFIX.length()));
return baseDir + VERSION_PREFIX + timestamp;
}
private Path getWorkingFlagPath(Path basePath) {
- long timestamp = pathToVersion.getOrDefault(basePath.toString(), System.currentTimeMillis());
+ return getWorkingFlagPath(basePath, System.currentTimeMillis());
+ }
+
+ private Path getWorkingFlagPath(Path basePath, long buildVersion) {
+ long timestamp = pathToVersion.getOrDefault(basePath.toString(), buildVersion);
pathToVersion.putIfAbsent(basePath.toString(), timestamp);
try {
FileSystem fileSystem = HadoopUtil.getFileSystem(basePath);
@@ -100,10 +104,10 @@ public class NGlobalDictS3Store extends NGlobalDictHDFSStore {
} catch (IOException ioe) {
logger.error("Get exception when get version", ioe);
}
- return getWorkingFlagPath(basePath, timestamp);
+ return generateWorkingFlagPath(basePath, timestamp);
}
- private Path getWorkingFlagPath(Path basePath, long version) {
+ private Path generateWorkingFlagPath(Path basePath, long version) {
return new Path(basePath + "/" + WORKING_PREFIX + version);
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictStore.java b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictStore.java
index 3a31d6dfb7..fd45d4c72b 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictStore.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictStore.java
@@ -48,7 +48,7 @@ public interface NGlobalDictStore {
void writeMetaInfo(int bucketSize, String workingDir) throws IOException;
- void commit(String workingDir, int maxVersions, long versionTTL) throws IOException;
+ void commit(String workingDir, int maxVersions, long versionTTL, long buildVersion) throws IOException;
- String getWorkingDir();
+ String getWorkingDir(long buildVersion);
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictionaryV2.java b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictionaryV2.java
index 8f37e43db2..769ca389e4 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictionaryV2.java
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NGlobalDictionaryV2.java
@@ -19,6 +19,9 @@ package org.apache.spark.dict;
import java.io.IOException;
import java.io.Serializable;
+import java.util.Arrays;
+
+import com.alibaba.nacos.common.JustForTest;
import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
@@ -29,6 +32,10 @@ import lombok.val;
public class NGlobalDictionaryV2 implements Serializable {
public static final String SEPARATOR = "_0_DOT_0_";
+
+ // The latest dictionary file is read by default if no version is specified
+ public static final long NO_VERSION_SPECIFIED = -1L;
+
protected static final Logger logger = LoggerFactory.getLogger(NGlobalDictionaryV2.class);
private NGlobalDictMetaInfo metadata;
@@ -36,27 +43,41 @@ public class NGlobalDictionaryV2 implements Serializable {
private String project;
private String sourceTable;
private String sourceColumn;
+
+ // Default is -1L, which means that no version is specified.
+ // Dict data is read from the latest version each time
+ private final long buildVersion;
private boolean isFirst = true;
- public NGlobalDictionaryV2(String project, String sourceTable, String sourceColumn, String baseDir)
- throws IOException {
- this.project = project;
- this.sourceTable = sourceTable;
- this.sourceColumn = sourceColumn;
+ public NGlobalDictionaryV2(String dictParams, long buildVersion) throws IOException {
+ val dictInfo = dictParams.split(SEPARATOR);
+ this.project = dictInfo[0];
+ this.sourceTable = dictInfo[1];
+ this.sourceColumn = dictInfo[2];
+ this.baseDir = dictInfo[3];
this.baseDir = baseDir + getResourceDir();
+ this.buildVersion = buildVersion;
this.metadata = getMetaInfo();
if (metadata != null) {
isFirst = false;
}
}
- public NGlobalDictionaryV2(String dictParams) throws IOException {
- val dictInfo = dictParams.split(SEPARATOR);
- this.project = dictInfo[0];
- this.sourceTable = dictInfo[1];
- this.sourceColumn = dictInfo[2];
- this.baseDir = dictInfo[3];
+ // Note: You do not need to specify a build version
+ // if you do not need to encode flat tables or
+ // if you only need to read the latest dictionary data
+ public NGlobalDictionaryV2(String project, String sourceTable, String sourceColumn, String baseDir)
+ throws IOException {
+ this(project,sourceTable,sourceColumn,baseDir, NO_VERSION_SPECIFIED);
+ }
+
+ public NGlobalDictionaryV2(String project, String sourceTable, String sourceColumn, String baseDir, long buildVersion)
+ throws IOException {
+ this.project = project;
+ this.sourceTable = sourceTable;
+ this.sourceColumn = sourceColumn;
this.baseDir = baseDir + getResourceDir();
+ this.buildVersion = buildVersion;
this.metadata = getMetaInfo();
if (metadata != null) {
isFirst = false;
@@ -68,18 +89,31 @@ public class NGlobalDictionaryV2 implements Serializable {
}
private String getWorkingDir() {
- return getResourceStore(baseDir).getWorkingDir();
+ return getResourceStore(baseDir).getWorkingDir(this.buildVersion);
}
public NBucketDictionary loadBucketDictionary(int bucketId) throws IOException {
return loadBucketDictionary(bucketId, false);
}
+ // Only for S3 test !
+ @JustForTest
+ public NBucketDictionary loadBucketDictionaryForTestS3(int bucketId) throws IOException {
+ if (null == metadata) {
+ metadata = getMetaInfoForTestS3();
+ }
+ return new NBucketDictionary(baseDir, getWorkingDir(), bucketId, metadata, false, this.buildVersion, true);
+ }
+
public NBucketDictionary loadBucketDictionary(int bucketId, boolean isForColumnEncoding) throws IOException {
+ return loadBucketDictionary(bucketId, isForColumnEncoding, this.buildVersion);
+ }
+
+ public NBucketDictionary loadBucketDictionary(int bucketId, boolean isForColumnEncoding, long buildVersion) throws IOException {
if (null == metadata) {
metadata = getMetaInfo();
}
- return new NBucketDictionary(baseDir, getWorkingDir(), bucketId, metadata, isForColumnEncoding);
+ return new NBucketDictionary(baseDir, getWorkingDir(), bucketId, metadata, isForColumnEncoding, buildVersion);
}
public NBucketDictionary createNewBucketDictionary() {
@@ -91,25 +125,56 @@ public class NGlobalDictionaryV2 implements Serializable {
globalDictStore.prepareForWrite(getWorkingDir());
}
+ // Only for S3 test !
+ @JustForTest
+ public void prepareWriteS3() throws IOException {
+ NGlobalDictStore globalDictStore = new NGlobalDictS3Store(baseDir);
+ globalDictStore.prepareForWrite(globalDictStore.getWorkingDir(this.buildVersion));
+ }
+
+ // Only for S3 test !
+ @JustForTest
+ public void writeMetaDictForTestS3(int bucketSize, int maxVersions, long versionTTL) throws IOException {
+ NGlobalDictStore globalDictStore = new NGlobalDictS3Store(baseDir);
+ String workingDir = globalDictStore.getWorkingDir(this.buildVersion);
+ globalDictStore.writeMetaInfo(bucketSize, workingDir);
+ globalDictStore.commit(workingDir, maxVersions, versionTTL, this.buildVersion);
+ }
+
public void writeMetaDict(int bucketSize, int maxVersions, long versionTTL) throws IOException {
NGlobalDictStore globalDictStore = getResourceStore(baseDir);
globalDictStore.writeMetaInfo(bucketSize, getWorkingDir());
commit(maxVersions, versionTTL);
}
- public NGlobalDictMetaInfo getMetaInfo() throws IOException {
- NGlobalDictStore globalDictStore = getResourceStore(baseDir);
- NGlobalDictMetaInfo metadata;
+
+ private NGlobalDictMetaInfo getMetaInfo(NGlobalDictStore globalDictStore) throws IOException {
+ NGlobalDictMetaInfo nGlobalDictMetaInfo;
Long[] versions = globalDictStore.listAllVersions();
logger.info("getMetaInfo versions.length is {}", versions.length);
-
if (versions.length == 0) {
return null;
+ } else if (this.buildVersion == NO_VERSION_SPECIFIED || !Arrays.asList(versions).contains(buildVersion)) {
+ logger.info("Initializes dict metainfo with the latest version:{}", versions[versions.length - 1]);
+ nGlobalDictMetaInfo = globalDictStore.getMetaInfo(versions[versions.length - 1]);
} else {
- metadata = globalDictStore.getMetaInfo(versions[versions.length - 1]);
+ logger.info("Initializes dict metainfo with the specified version:{}", this.buildVersion);
+ nGlobalDictMetaInfo = globalDictStore.getMetaInfo(this.buildVersion);
}
- logger.info("getMetaInfo metadata is null : [{}]", metadata == null);
- return metadata;
+ logger.info("getMetaInfo metadata is null : [{}]", nGlobalDictMetaInfo == null);
+ return nGlobalDictMetaInfo;
+ }
+
+ // Only for S3 test !
+ @JustForTest
+ public NGlobalDictMetaInfo getMetaInfoForTestS3() throws IOException {
+ NGlobalDictStore globalDictStore = new NGlobalDictS3Store(baseDir);
+ return getMetaInfo(globalDictStore);
+ }
+
+ public NGlobalDictMetaInfo getMetaInfo() throws IOException {
+ NGlobalDictStore globalDictStore = getResourceStore(baseDir);
+ return getMetaInfo(globalDictStore);
}
public int getBucketSizeOrDefault(int defaultSize) {
@@ -133,7 +198,7 @@ public class NGlobalDictionaryV2 implements Serializable {
private void commit(int maxVersions, long versionTTL) throws IOException {
NGlobalDictStore globalDictStore = getResourceStore(baseDir);
- globalDictStore.commit(getWorkingDir(), maxVersions, versionTTL);
+ globalDictStore.commit(getWorkingDir(), maxVersions, versionTTL, this.buildVersion);
}
private NGlobalDictStore getResourceStore(String baseDir) {
(kylin) 21/35: KYLIN-5817 Query scanRows and bytes are incorrect
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 00352e8c2d53307b4cf3f847fc6bbf2a869c4520
Author: haocheni <ha...@qq.com>
AuthorDate: Fri Sep 22 10:29:48 2023 +0800
KYLIN-5817 Query scanRows and bytes are incorrect
---
.../src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala | 4 ++--
.../org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
index 4beda86353..68fc7b9b19 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
@@ -63,8 +63,8 @@ object QueryMetricUtils extends Logging {
child => {
if (child.isInstanceOf[SparkPlan]) {
val result = collectAdaptiveSparkPlanExecMetrics(child, scanRow, scanBytes)
- newScanRow = result._1
- newScanBytes = result._2
+ newScanRow += result._1
+ newScanBytes += result._2
} else {
logTrace("Not sparkPlan in collectAdaptiveSparkPlanExecMetrics, child: " + child.getClass.getName)
}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
index ede02eef60..f0eda5b817 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
@@ -217,8 +217,8 @@ class SparkQueryMetricUtilsSuite extends QueryTest with SharedSparkSession {
assert(0 == collectScanMetrics._1.get(0))
assert(0 == collectScanMetrics._2.get(0))
val collectScanMetrics2 = QueryMetricUtils.collectAdaptiveSparkPlanExecMetrics(dataWritingCommandExec, 1, 1)
- assert(1 == collectScanMetrics2._1)
- assert(1 == collectScanMetrics2._2)
+ assert(2 == collectScanMetrics2._1)
+ assert(2 == collectScanMetrics2._2)
}
(kylin) 08/35: KYLIN-5800 & KYLIN-5812 Support scalar subquery join aggregate pushdown
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit dc344ce2fff2b2ca1ece1955b98c4e4130fecbd3
Author: Yu Gan <yu...@kyligence.io>
AuthorDate: Fri Aug 25 16:01:23 2023 +0800
KYLIN-5800 & KYLIN-5812 Support scalar subquery join aggregate pushdown
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +
.../query/rules/ScalarSubqueryJoinRuleTest.java | 112 +++
.../query/rules/ScalarSubqueryJoinRuleTest.xml | 778 ++++++++++++++++++++
.../query/sql_scalar_subquery/query01.sql | 26 +
.../query/sql_scalar_subquery/query02.sql | 30 +
.../query/sql_scalar_subquery/query03.sql | 32 +
.../query/sql_scalar_subquery/query04.sql | 28 +
.../query/sql_scalar_subquery/query05.sql | 37 +
.../query/sql_scalar_subquery/query06.sql | 35 +
.../query/sql_scalar_subquery/query07.sql | 33 +
.../query/sql_scalar_subquery/query08.sql | 36 +
.../query/sql_scalar_subquery/query09.sql | 36 +
.../query/sql_scalar_subquery/query10.sql | 33 +
.../query/sql_scalar_subquery/query11.sql | 32 +
.../query/sql_scalar_subquery/query12.sql | 41 ++
.../query/sql_scalar_subquery/query13.sql | 36 +
.../query/sql_scalar_subquery/query14.sql | 32 +
.../query/sql_scalar_subquery/query15.sql | 27 +
.../query/sql_scalar_subquery/query16.sql | 33 +
.../query/sql_scalar_subquery/query17.sql | 31 +
.../query/sql_scalar_subquery/query18.sql | 65 ++
.../query/sql_scalar_subquery/query19.sql | 26 +
.../apache/kylin/query/relnode/KapValuesRel.java | 21 +-
.../apache/kylin/query/relnode/OLAPValuesRel.java | 13 +-
.../kyligence/kap/query/optrule/KAPValuesRule.java | 11 +-
.../kap/query/optrule/ScalarSubqueryJoinRule.java | 799 +++++++++++++++++++++
.../org/apache/kylin/query/engine/QueryExec.java | 4 +
.../java/org/apache/kylin/query/util/HepUtils.java | 20 +-
28 files changed, 2393 insertions(+), 18 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8b9a0308f7..64b4a775d4 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2532,6 +2532,10 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.query.calcite.aggregate-pushdown-enabled", FALSE));
}
+ public boolean isScalarSubqueryJoinEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.scalar-subquery-join-enabled", FALSE));
+ }
+
public int getCalciteBindableCacheSize() {
return Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.maxSize", "10"));
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java
new file mode 100644
index 0000000000..0c8c75ac89
--- /dev/null
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.rules;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
+import org.apache.calcite.test.DiffRepository;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import io.kyligence.kap.query.optrule.AggregateProjectReduceRule;
+import io.kyligence.kap.query.optrule.KapAggregateRule;
+import io.kyligence.kap.query.optrule.KapJoinRule;
+import io.kyligence.kap.query.optrule.KapProjectRule;
+import io.kyligence.kap.query.optrule.ScalarSubqueryJoinRule;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ScalarSubqueryJoinRuleTest extends CalciteRuleTestBase {
+
+ private final String project = "default";
+ private final String sqlFolder = "query/sql_scalar_subquery";
+ //
+ private final DiffRepository diff = DiffRepository.lookup(ScalarSubqueryJoinRuleTest.class);
+
+ @Before
+ public void setup() {
+ createTestMetadata();
+ }
+
+ @After
+ public void cleanUp() {
+ cleanupTestMetadata();
+ }
+
+ @Override
+ protected DiffRepository getDiffRepo() {
+ return diff;
+ }
+
+ @Test
+ public void testPushDownAggregate() throws IOException {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ List<Pair<String, String>> queries = readALLSQLs(config, project, sqlFolder);
+ List<RelOptRule> rulesToAdd = getTransformRules();
+
+ queries.forEach(p -> doCheckTransform(p.getFirst(), p.getSecond(), rulesToAdd));
+ }
+
+ private void doCheckTransform(String prefix, String sql, List<RelOptRule> rulesToAdd) {
+ checkSQLPostOptimize(project, sql, prefix, null, rulesToAdd);
+ }
+
+ @Test
+ @Ignore("Dev only")
+ public void testSingle() throws IOException {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ List<RelOptRule> rulesToAdd = getTransformRules();
+ Pair<String, String> q = readOneSQL(config, project, sqlFolder, "query03.sql");
+ doCheckTransform(q.getFirst(), q.getSecond(), rulesToAdd);
+ }
+
+ @Test
+ @Ignore("Dev only")
+ public void dumpPlans() throws IOException {
+ List<Pair<String, String>> queries = readALLSQLs(KylinConfig.getInstanceFromEnv(), project, sqlFolder);
+ CalciteRuleTestBase.StringOutput output = new CalciteRuleTestBase.StringOutput(false);
+ final List<RelOptRule> rulesToAdd = getTransformRules();
+ queries.forEach(p -> checkSQLPostOptimize(project, p.getSecond(), p.getFirst(), output, rulesToAdd));
+ output.dump(log);
+ }
+
+ private List<RelOptRule> getTransformRules() {
+ return ImmutableList.of(// basic rules
+ KapAggregateRule.INSTANCE, //
+ KapProjectRule.INSTANCE, //
+ KapJoinRule.INSTANCE, //
+ // relative rules
+ ProjectMergeRule.INSTANCE, //
+ AggregateProjectMergeRule.INSTANCE, //
+ AggregateProjectReduceRule.INSTANCE, //
+ // target rules
+ ScalarSubqueryJoinRule.AGG_JOIN, //
+ ScalarSubqueryJoinRule.AGG_PRJ_JOIN, //
+ ScalarSubqueryJoinRule.AGG_PRJ_FLT_JOIN);
+ }
+}
diff --git a/src/kylin-it/src/test/resources/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.xml b/src/kylin-it/src/test/resources/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.xml
new file mode 100644
index 0000000000..55c9a36a16
--- /dev/null
+++ b/src/kylin-it/src/test/resources/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.xml
@@ -0,0 +1,778 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Root>
+ <TestCase name="testPushDownAggregate">
+ <Resource name="query01.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], TOTAL=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATEKEY=[$1], DATE_TIME=[$0], D_YEAR=[$5], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query01.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], DATE_TIME=[$0], TOTAL=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], TOTAL=[SUM($2)], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0]], groups=[null], TOTAL=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query02.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], TOTAL=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATEKEY=[$1], DATE_TIME=[$0], D_YEAR=[$5], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query02.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], DATE_TIME=[$0], TOTAL=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], TOTAL=[SUM($2)], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0]], groups=[null], TOTAL=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query03.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATE=[$2], DATE_TIME=[$0], D_YEAR=[$5], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($2, $0), >=($2, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query03.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATE=[$1], DATE_TIME=[$0], EXPR$2=[$2], SY=[$3], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[$SUM0($2)], SY=[SUM($3)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATE=[$1], $f2=[CASE(IS NULL($2), CAST(1):BIGINT, $2)], SY=[$3], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($1, $0), >=($1, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query04.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$0], TOTAL=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], TOTAL=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATEKEY=[$1], DATE_TIME=[$0], D_YEAR=[$5], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query04.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], TOTAL=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], TOTAL=[SUM($2)], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0]], groups=[null], TOTAL=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query05.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(SD=[$3], D_DATE=[$0], EXPR$2=[$4], DATE_TIME=[$1], SY=[$5], SR=[$6], LO_ORDERDATE=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2]], groups=[null], SD=[SUM($3)], EXPR$2=[COUNT()], SY=[SUM($4)], SR=[SUM($5)], ctx=[])
+ KapProjectRel(D_DATE=[$2], DATE_TIME=[$0], LO_ORDERDATE=[$23], D_DATEKEY=[$1], D_YEAR=[$5], LO_REVENUE=[$30], ctx=[])
+ KapJoinRel(condition=[=($0, $23)], joinType=[inner], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($2, $0), >=($2, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+]]>
+ </Resource>
+ <Resource name="query05.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(SD=[$3], D_DATE=[$1], EXPR$2=[$4], DATE_TIME=[$0], SY=[$5], SR=[$6], LO_ORDERDATE=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2]], groups=[null], SD=[SUM($8)], EXPR$2=[$SUM0($9)], SY=[SUM($10)], SR=[SUM($11)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATE=[$1], LO_ORDERDATE=[$5], EXPR$2=[$3], SY=[$4], SD=[$2], $f1=[$6], SR=[$7], $f8=[CAST(*($2, $6)):BIGINT], $f9=[*($3, $6)], $f10=[CAST(*($4, $6)):BIGINT], $f11=[CAST(*($3, $7)):BIGINT], ctx=[])
+ KapJoinRel(condition=[=($0, $5)], joinType=[inner], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], SD=[SUM($2)], EXPR$2=[$SUM0($3)], SY=[SUM($4)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATE=[$1], SD=[$2], $f3=[CASE(IS NULL($3), CAST(1):BIGINT, $3)], SY=[$4], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($1, $0), >=($1, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[1]], groups=[null], SD=[SUM($0)], EXPR$2=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapAggregateRel(group-set=[[5]], groups=[null], agg#0=[COUNT()], SR=[SUM($12)], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+]]>
+ </Resource>
+ <Resource name="query06.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATE=[$4], DATE_TIME=[$0], D_YEAR=[$2], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($4, $0), >=($4, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(D_DATEKEY=[$0], D_YEAR=[$4], D_HOLIDAYFL=[$15], D_DATE=[$1], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query06.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATE=[$1], DATE_TIME=[$0], EXPR$2=[$2], SY=[$3], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[$SUM0($2)], SY=[SUM($3)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATE=[$1], $f2=[CASE(IS NULL($2), CAST(1):BIGINT, $2)], SY=[$3], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($1, $0), >=($1, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query07.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATE=[$1], DATE_TIME=[$17], D_YEAR=[$4], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($1, $17), >=($1, CONCAT(SUBSTRING($17, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query07.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 3]], groups=[null], EXPR$2=[$SUM0($1)], SY=[SUM($2)], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($0, $3), >=($0, CONCAT(SUBSTRING($3, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapAggregateRel(group-set=[[1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query08.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATE=[$3], DATE_TIME=[$4], D_YEAR=[$1], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($3, $4), >=($3, CONCAT(SUBSTRING($4, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapProjectRel(D_DATEKEY=[$0], D_YEAR=[$4], D_HOLIDAYFL=[$15], D_DATE=[$1], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query08.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 3]], groups=[null], EXPR$2=[$SUM0($1)], SY=[SUM($2)], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($0, $3), >=($0, CONCAT(SUBSTRING($3, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapAggregateRel(group-set=[[1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query09.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATE=[$4], DATE_TIME=[$0], D_YEAR=[$3], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($4, $0), >=($4, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(D_DATEKEY=[$0], EXPR$1=[$3], D_YEAR=[$2], D_DATE=[$1], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2]], groups=[null], EXPR$1=[SUM($2)], ctx=[])
+ KapProjectRel(D_DATEKEY=[$0], D_DATE=[$1], D_YEAR=[$4], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query09.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATE=[$1], DATE_TIME=[$0], EXPR$2=[$2], SY=[$3], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$2=[$SUM0($2)], SY=[SUM($3)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATE=[$1], $f2=[CASE(IS NULL($2), CAST(1):BIGINT, $2)], SY=[$3], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($1, $0), >=($1, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[1]], groups=[null], EXPR$2=[COUNT()], SY=[SUM($2)], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 4]], groups=[null], EXPR$1=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query10.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(EXPR$0=[$1], SY=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], EXPR$0=[COUNT()], SY=[SUM($1)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_YEAR=[$5], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=(CAST($1):BIGINT, CAST($0):BIGINT NOT NULL), >=(CAST($1):BIGINT, CAST(CONCAT(SUBSTRING($0, 1, 8), '01')):BIGINT NOT NULL))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['2300-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query10.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(EXPR$0=[$1], SY=[$2], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], EXPR$0=[$SUM0($2)], SY=[SUM($3)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATEKEY=[$1], $f2=[CASE(IS NULL($2), CAST(1):BIGINT, $2)], SY=[$3], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=(CAST($1):BIGINT, CAST($0):BIGINT NOT NULL), >=(CAST($1):BIGINT, CAST(CONCAT(SUBSTRING($0, 1, 8), '01')):BIGINT NOT NULL))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['2300-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0]], groups=[null], EXPR$0=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query11.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[COUNT()], SY=[SUM($0)], ctx=[])
+ KapProjectRel(D_YEAR=[$5], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=(CAST($1):BIGINT, CAST($0):BIGINT NOT NULL), >=(CAST($1):BIGINT, CAST(CONCAT(SUBSTRING($0, 1, 8), '01')):BIGINT NOT NULL))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['2300-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query11.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[$SUM0($2)], SY=[SUM($3)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATEKEY=[$1], $f2=[CASE(IS NULL($2), CAST(1):BIGINT, $2)], SY=[$3], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=(CAST($1):BIGINT, CAST($0):BIGINT NOT NULL), >=(CAST($1):BIGINT, CAST(CONCAT(SUBSTRING($0, 1, 8), '01')):BIGINT NOT NULL))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['2300-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0]], groups=[null], EXPR$0=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query12.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(DATE_TIME=[$0], EXPR$1=[$1], EXPR$2=[$2], EXPR$3=[$3], EXPR$4=[$3], EXPR$5=[$4], SY=[$5], SR=[$6], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], EXPR$1=[COUNT($1, $2)], EXPR$2=[COUNT($1)], EXPR$3=[COUNT()], EXPR$5=[COUNT($2)], SY=[SUM($3)], SR=[SUM($4)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_HOLIDAYFL=[$16], LO_CUSTKEY=[$20], D_YEAR=[$5], LO_REVENUE=[$30], ctx=[])
+ KapJoinRel(condition=[=($0, $23)], joinType=[inner], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($2, $0), >=($2, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['2050-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+]]>
+ </Resource>
+ <Resource name="query12.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(DATE_TIME=[$0], EXPR$1=[$1], EXPR$2=[$2], EXPR$3=[$3], EXPR$4=[$3], EXPR$5=[$4], SY=[$5], SR=[$6], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], EXPR$1=[$SUM0($8)], EXPR$2=[$SUM0($9)], EXPR$3=[$SUM0($10)], EXPR$5=[$SUM0($11)], SY=[SUM($12)], SR=[SUM($13)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], $f1=[$1], EXPR$3=[$2], SY=[$3], LO_ORDERDATE=[$4], $f10=[$5], $f2=[$6], SR=[$7], $f8=[*($1, $5)], $f9=[*($1, $6)], $f1011=[*($2, $6)], $f11=[*($2, $5)], $f12=[CAST(*($3, $6)):BIGINT], $f13=[CAST(*($2, $7)):BIGINT], ctx=[])
+ KapJoinRel(condition=[=($0, $4)], joinType=[inner], ctx=[])
+ KapAggregateRel(group-set=[[0]], groups=[null], agg#0=[$SUM0($2)], EXPR$3=[$SUM0($3)], SY=[SUM($4)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATE=[$1], $f2=[CASE(IS NULL($2), CAST(0):BIGINT, $2)], $f3=[CASE(IS NULL($3), CAST(1):BIGINT, $3)], SY=[$4], ctx=[])
+ KapNonEquiJoinRel(condition=[AND(<=($1, $0), >=($1, CONCAT(SUBSTRING($0, 1, 8), '01')))], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapProjectRel(DATE_TIME=['1995-03-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['2050-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-03'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[1]], groups=[null], agg#0=[COUNT($15)], EXPR$3=[COUNT()], SY=[SUM($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapAggregateRel(group-set=[[5]], groups=[null], agg#0=[COUNT($2)], agg#1=[COUNT()], SR=[SUM($12)], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+]]>
+ </Resource>
+ <Resource name="query13.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$2], MIN_DAYOFWEEK=[$3], DATE_TIME=[$1], D_DATE=[$0], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2]], groups=[null], MIN_DAYOFWEEK=[COUNT($3)], ctx=[])
+ KapProjectRel(D_DATE=[$1], DATE_TIME=[$17], D_DATEKEY=[$0], D_DAYOFWEEK=[$2], ctx=[])
+ KapJoinRel(condition=[=($0, $17)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-05-17'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query13.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$0], MIN_DAYOFWEEK=[$3], DATE_TIME=[$2], D_DATE=[$1], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 3]], groups=[null], MIN_DAYOFWEEK=[$SUM0($2)], ctx=[])
+ KapJoinRel(condition=[=($0, $3)], joinType=[inner], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], MIN_DAYOFWEEK=[COUNT($2)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-05-17'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query14.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], SUM_YEAR=[$2], DATE_TIME=[$0], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], SUM_YEAR=[COUNT($2)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATEKEY=[$1], D_YEAR=[$5], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-05-17'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query14.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], SUM_YEAR=[$2], DATE_TIME=[$0], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], SUM_YEAR=[$SUM0($2)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATEKEY=[$1], $f2=[CASE(IS NULL($2), CAST(0):BIGINT, $2)], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-05-17'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0]], groups=[null], SUM_YEAR=[COUNT($4)], ctx=[])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query15.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(C_NAME=[$0], COUNT_ORDER=[$2], LO_ORDERDATE=[$1], SUM_CUSTKEY=[$3], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], COUNT_ORDER=[COUNT($2)], SUM_CUSTKEY=[SUM($3)], ctx=[])
+ KapProjectRel(C_NAME=[$22], LO_ORDERDATE=[$5], LO_ORDERKEY=[$0], C_CUSTKEY=[$21], ctx=[])
+ KapJoinRel(condition=[=($2, $21)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+ KapTableScan(table=[[SSB, CUSTOMER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])
+]]>
+ </Resource>
+ <Resource name="query15.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(C_NAME=[$1], COUNT_ORDER=[$2], LO_ORDERDATE=[$0], SUM_CUSTKEY=[$3], ctx=[])
+ KapAggregateRel(group-set=[[5, 22]], groups=[null], COUNT_ORDER=[COUNT($0)], SUM_CUSTKEY=[SUM($21)], ctx=[])
+ KapJoinRel(condition=[=($2, $21)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+ KapTableScan(table=[[SSB, CUSTOMER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])
+]]>
+ </Resource>
+ <Resource name="query16.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], COUNT_YEAR=[$2], DATE_TIME=[$0], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], COUNT_YEAR=[COUNT($2)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], D_DATEKEY=[$1], D_YEAR=[$5], ctx=[])
+ KapFilterRel(condition=[=($4, 'May')], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-05-17'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query16.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(D_DATEKEY=[$1], COUNT_YEAR=[$2], DATE_TIME=[$0], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], COUNT_YEAR=[COUNT($5)], ctx=[])
+ KapFilterRel(condition=[=($4, 'May')], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-05-17'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapTableScan(table=[[SSB, DATES]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]])
+]]>
+ </Resource>
+ <Resource name="query17.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(C_NAME=[$0], COUNT_ORDER=[$3], LO_ORDERDATE=[$1], DATE_TIME=[$2], SUM_CUSTKEY=[$4], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2]], groups=[null], COUNT_ORDER=[COUNT($3)], SUM_CUSTKEY=[SUM($4)], ctx=[])
+ KapProjectRel(C_NAME=[$22], LO_ORDERDATE=[$5], DATE_TIME=[$29], LO_ORDERKEY=[$0], C_CUSTKEY=[$21], ctx=[])
+ KapJoinRel(condition=[=($5, $29)], joinType=[inner], ctx=[])
+ KapJoinRel(condition=[=($2, $21)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+ KapTableScan(table=[[SSB, CUSTOMER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query17.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(C_NAME=[$1], COUNT_ORDER=[$3], LO_ORDERDATE=[$0], DATE_TIME=[$2], SUM_CUSTKEY=[$4], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 4]], groups=[null], COUNT_ORDER=[$SUM0($2)], SUM_CUSTKEY=[SUM($3)], ctx=[])
+ KapJoinRel(condition=[=($0, $4)], joinType=[inner], ctx=[])
+ KapAggregateRel(group-set=[[5, 22]], groups=[null], COUNT_ORDER=[COUNT($0)], SUM_CUSTKEY=[SUM($21)], ctx=[])
+ KapJoinRel(condition=[=($2, $21)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+ KapTableScan(table=[[SSB, CUSTOMER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query18.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(C_NAME=[$0], COUNT_ORDER=[$4], LO_ORDERDATE=[$1], DATE_TIME=[$2], DATE_TIME0=[$3], SUM_CUSTKEY=[$5], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2, 3]], groups=[null], COUNT_ORDER=[COUNT($4)], SUM_CUSTKEY=[SUM($5)], ctx=[])
+ KapProjectRel(C_NAME=[$3], LO_ORDERDATE=[$1], DATE_TIME=[$0], $f3=[$5], LO_ORDERKEY=[$2], C_CUSTKEY=[$4], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(LO_ORDERDATE=[$0], LO_ORDERKEY=[$1], C_NAME=[$2], C_CUSTKEY=[$3], DATE_TIME=[$4], ctx=[])
+ KapProjectRel(LO_ORDERDATE=[$0], LO_ORDERKEY=[$1], C_NAME=[$2], C_CUSTKEY=[$3], DATE_TIME=[$5], ctx=[])
+ KapJoinRel(condition=[=($0, $5)], joinType=[inner], ctx=[])
+ KapProjectRel(LO_ORDERDATE=[$5], LO_ORDERKEY=[$0], C_NAME=[$22], C_CUSTKEY=[$21], DATE_TIME=[$29], ctx=[])
+ KapJoinRel(condition=[=($5, $29)], joinType=[inner], ctx=[])
+ KapJoinRel(condition=[=($2, $21)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+ KapTableScan(table=[[SSB, CUSTOMER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-06-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query18.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(C_NAME=[$2], COUNT_ORDER=[$4], LO_ORDERDATE=[$1], DATE_TIME=[$0], DATE_TIME0=[$3], SUM_CUSTKEY=[$5], ctx=[])
+ KapAggregateRel(group-set=[[0, 1, 2, 3]], groups=[null], COUNT_ORDER=[$SUM0($4)], SUM_CUSTKEY=[SUM($5)], ctx=[])
+ KapProjectRel(DATE_TIME=[$0], LO_ORDERDATE=[$1], C_NAME=[$2], DATE_TIME0=[$3], $f4=[CASE(IS NULL($4), CAST(0):BIGINT, $4)], SUM_CUSTKEY=[$5], ctx=[])
+ KapJoinRel(condition=[=($0, $1)], joinType=[left], ctx=[])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapAggregateRel(group-set=[[0, 1, 4]], groups=[null], COUNT_ORDER=[$SUM0($2)], SUM_CUSTKEY=[SUM($3)], ctx=[])
+ KapJoinRel(condition=[=($0, $4)], joinType=[inner], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], COUNT_ORDER=[$SUM0($2)], SUM_CUSTKEY=[SUM($3)], ctx=[])
+ KapJoinRel(condition=[=($0, $4)], joinType=[inner], ctx=[])
+ KapAggregateRel(group-set=[[5, 22]], groups=[null], COUNT_ORDER=[COUNT($0)], SUM_CUSTKEY=[SUM($21)], ctx=[])
+ KapJoinRel(condition=[=($2, $21)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]])
+ KapTableScan(table=[[SSB, CUSTOMER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7]])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[true], ctx=[], all=[true])
+ KapUnionRel(all=[false], ctx=[], all=[false])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-03-29'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-04-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1995-06-01'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+ KapProjectRel(DATE_TIME=['1996-01-02'], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query19.planBefore">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(EXPR$0=[$2], LO_ORDERDATE=[$0], DATE_TIME=[$1], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$0=[SUM($2)], ctx=[])
+ KapProjectRel(LO_ORDERDATE=[$5], DATE_TIME=[$18], $f2=[*($9, 3)], ctx=[])
+ KapJoinRel(condition=[=($5, $18)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+ KapProjectRel(DATE_TIME=[19931014], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+ <Resource name="query19.planAfter">
+ <![CDATA[
+KapOLAPToEnumerableConverter
+ KapProjectRel(EXPR$0=[$2], LO_ORDERDATE=[$0], DATE_TIME=[$1], ctx=[])
+ KapAggregateRel(group-set=[[0, 1]], groups=[null], EXPR$0=[SUM($2)], ctx=[])
+ KapProjectRel(LO_ORDERDATE=[$5], DATE_TIME=[$18], $f2=[*($9, 3)], ctx=[])
+ KapJoinRel(condition=[=($5, $18)], joinType=[inner], ctx=[])
+ KapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+ KapProjectRel(DATE_TIME=[19931014], ctx=[])
+ KapValuesRel(tuples=[[{ 0 }]])
+]]>
+ </Resource>
+
+ </TestCase>
+</Root>
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query01.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query01.sql
new file mode 100644
index 0000000000..b757e10ec3
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query01.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATEKEY,
+ DATE_TIME,
+ sum(D_YEAR) total
+FROM (SELECT '1995-03-01' AS DATE_TIME) dt
+ LEFT JOIN SSB.DATES t2 ON dt.DATE_TIME = t2.D_DATEKEY
+GROUP BY D_DATEKEY,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query02.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query02.sql
new file mode 100644
index 0000000000..dec0698161
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query02.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATEKEY,
+ DATE_TIME,
+ sum(D_YEAR) total
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) dt
+ LEFT JOIN SSB.DATES t2 ON dt.DATE_TIME = t2.D_DATEKEY
+GROUP BY D_DATEKEY,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query03.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query03.sql
new file mode 100644
index 0000000000..a7221b72cc
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query03.sql
@@ -0,0 +1,32 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATE,
+ DATE_TIME,
+ count(*),
+ sum(D_YEAR) sy
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN SSB.DATES t2 ON t2.D_DATE <= t1.DATE_TIME
+ AND t2.D_DATE >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
+GROUP BY D_DATE,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query04.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query04.sql
new file mode 100644
index 0000000000..30d8a26b5c
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query04.sql
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATEKEY,
+ total
+FROM (SELECT D_DATEKEY,
+ DATE_TIME,
+ sum(D_YEAR) total
+ FROM (SELECT '1995-03-01' AS DATE_TIME) dt
+ LEFT JOIN SSB.DATES t2 ON dt.DATE_TIME = t2.D_DATEKEY
+ GROUP BY D_DATEKEY,
+ DATE_TIME)
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query05.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query05.sql
new file mode 100644
index 0000000000..710afa2e03
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query05.sql
@@ -0,0 +1,37 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT sum(D_DATEKEY) as sd,
+ D_DATE,
+ count(*),
+ DATE_TIME,
+ sum(D_YEAR) sy,
+ sum(LO_REVENUE) sr,
+ LO_ORDERDATE
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN SSB.DATES t2 ON t2.D_DATE <= t1.DATE_TIME
+ AND t2.D_DATE >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
+ INNER JOIN SSB.LINEORDER t3 ON t3.LO_ORDERDATE = t1.DATE_TIME
+GROUP BY D_DATE,
+ DATE_TIME,
+ LO_ORDERDATE
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query06.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query06.sql
new file mode 100644
index 0000000000..9c765a7b12
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query06.sql
@@ -0,0 +1,35 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATE,
+ DATE_TIME,
+ count(*),
+ sum(D_YEAR) sy
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN (
+ SELECT D_DATEKEY, D_YEAR, D_HOLIDAYFL, D_DATE
+ from SSB.DATES
+) t2 ON t2.D_DATE <= t1.DATE_TIME
+ AND t2.D_DATE >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
+GROUP BY D_DATE,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query07.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query07.sql
new file mode 100644
index 0000000000..85ce225b0e
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query07.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATE,
+ DATE_TIME,
+ count(*),
+ sum(D_YEAR) sy
+FROM SSB.DATES t1
+ LEFT JOIN (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t2
+ ON t1.D_DATE <= t2.DATE_TIME
+ AND t1.D_DATE >= CONCAT(SUBSTR(t2.DATE_TIME, 1, 8), '01')
+GROUP BY D_DATE,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query08.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query08.sql
new file mode 100644
index 0000000000..2981e00b2c
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query08.sql
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATE,
+ DATE_TIME,
+ count(*),
+ sum(D_YEAR) sy
+FROM (
+ SELECT D_DATEKEY, D_YEAR, D_HOLIDAYFL, D_DATE
+ from SSB.DATES
+ ) t1
+ LEFT JOIN (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t2
+ ON t1.D_DATE <= t2.DATE_TIME
+ AND t1.D_DATE >= CONCAT(SUBSTR(t2.DATE_TIME, 1, 8), '01')
+GROUP BY D_DATE,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query09.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query09.sql
new file mode 100644
index 0000000000..571cb26cdd
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query09.sql
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT D_DATE,
+ DATE_TIME,
+ count(*),
+ sum(D_YEAR) sy
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN (
+ SELECT D_DATEKEY, SUM(D_YEAR), D_YEAR, D_DATE
+ from SSB.DATES
+ GROUP BY D_DATEKEY, D_DATE, D_YEAR
+) t2 ON t2.D_DATE <= t1.DATE_TIME
+ AND t2.D_DATE >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
+GROUP BY D_DATE,
+ DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query10.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query10.sql
new file mode 100644
index 0000000000..3db46727bc
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query10.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT count(*),
+ sum(D_YEAR) sy
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '2300-03-03' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN SSB.DATES t2 ON t2.D_DATEKEY <= t1.DATE_TIME
+ AND t2.D_DATEKEY >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
+GROUP BY DATE_TIME
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query11.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query11.sql
new file mode 100644
index 0000000000..29e3687de6
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query11.sql
@@ -0,0 +1,32 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT count(*),
+ sum(D_YEAR) sy
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '2300-03-03' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN SSB.DATES t2 ON t2.D_DATEKEY <= t1.DATE_TIME
+ AND t2.D_DATEKEY >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query12.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query12.sql
new file mode 100644
index 0000000000..218cc57a5d
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query12.sql
@@ -0,0 +1,41 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT DATE_TIME,
+ count(D_HOLIDAYFL, LO_CUSTKEY),
+ count(D_HOLIDAYFL),
+ count(*),
+ count(1),
+ count(LO_CUSTKEY),
+ sum(D_YEAR) sy,
+ sum(LO_REVENUE) sr
+FROM (SELECT '1995-03-01' AS DATE_TIME
+ UNION ALL
+ SELECT '2050-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-02' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-03' AS DATE_TIME) t1
+ LEFT JOIN SSB.DATES t2
+ ON t2.D_DATE <= t1.DATE_TIME
+ AND t2.D_DATE >= CONCAT(SUBSTR(t1.DATE_TIME, 1, 8), '01')
+ INNER JOIN SSB.LINEORDER t3 ON t3.LO_ORDERDATE = t1.DATE_TIME
+GROUP BY DATE_TIME
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query13.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query13.sql
new file mode 100644
index 0000000000..a026843a08
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query13.sql
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select DATES.D_DATEKEY,
+ count(DATES.D_DAYOFWEEK) min_dayofweek,
+ DT.DATE_TIME,
+ DATES.D_DATE
+from SSB.DATES
+ inner join
+ (select '1995-03-29' as DATE_TIME
+ union
+ select '1995-04-01' as DATE_TIME
+ union all
+ select '1995-03-29' as DATE_TIME
+ union all
+ select '1995-04-01' as DATE_TIME
+ union all
+ select '1995-05-17' as DATE_TIME) DT
+ on DT.DATE_TIME = DATES.D_DATEKEY
+group by DATES.D_DATE, date_time, DATES.D_DATEKEY
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query14.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query14.sql
new file mode 100644
index 0000000000..fea673ec28
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query14.sql
@@ -0,0 +1,32 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select DATES.D_DATEKEY, count(DATES.D_YEAR) sum_year, DT.DATE_TIME
+from (select '1995-03-29' as DATE_TIME
+ union
+ select '1995-04-01' as DATE_TIME
+ union all
+ select '1995-03-29' as DATE_TIME
+ union all
+ select '1995-04-01' as DATE_TIME
+ union all
+ select '1995-05-17' as DATE_TIME) DT
+ left join SSB.DATES
+ on DT.DATE_TIME = DATES.D_DATEKEY
+group by dt.date_time, DATES.D_DATEKEY
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query15.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query15.sql
new file mode 100644
index 0000000000..760bb2d3d2
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query15.sql
@@ -0,0 +1,27 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT CUSTOMER.C_NAME,
+ count(LINEORDER.LO_ORDERKEY) count_order,
+ LINEORDER.LO_ORDERDATE,
+ sum(CUSTOMER.C_CUSTKEY) sum_custkey
+FROM SSB.LINEORDER
+ INNER JOIN SSB.CUSTOMER ON LINEORDER.LO_CUSTKEY = CUSTOMER.C_CUSTKEY
+GROUP BY CUSTOMER.C_NAME,
+ LINEORDER.LO_ORDERDATE
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query16.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query16.sql
new file mode 100644
index 0000000000..1415b5025e
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query16.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select DATES.D_DATEKEY, count(DATES.D_YEAR) count_year, DT.DATE_TIME
+from (select '1995-03-29' as DATE_TIME
+ union
+ select '1995-04-01' as DATE_TIME
+ union all
+ select '1995-03-29' as DATE_TIME
+ union all
+ select '1995-04-01' as DATE_TIME
+ union all
+ select '1995-05-17' as DATE_TIME) DT
+ left join SSB.DATES on DT.DATE_TIME = DATES.D_DATEKEY
+where DATES.D_MONTH = 'May'
+group by dt.date_time, DATES.D_DATEKEY
+
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query17.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query17.sql
new file mode 100644
index 0000000000..7ca20a33a6
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query17.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT CUSTOMER.C_NAME,
+ count(LINEORDER.LO_ORDERKEY) count_order,
+ LINEORDER.LO_ORDERDATE,
+ dt.DATE_TIME,
+ sum(CUSTOMER.C_CUSTKEY) sum_custkey
+FROM "SSB"."LINEORDER" as "LINEORDER"
+ INNER JOIN "SSB"."CUSTOMER" as "CUSTOMER"
+ on "LINEORDER"."LO_CUSTKEY" = "CUSTOMER"."C_CUSTKEY"
+ inner join (select '1996-01-02' as DATE_TIME) dt
+ on LINEORDER.LO_ORDERDATE = dt.DATE_TIME
+group by CUSTOMER.C_NAME, LINEORDER.LO_ORDERDATE, dt.DATE_TIME
+
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query18.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query18.sql
new file mode 100644
index 0000000000..fc433c7e4b
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query18.sql
@@ -0,0 +1,65 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+SELECT li.C_NAME,
+ count(li.LO_ORDERKEY) count_order,
+ li.LO_ORDERDATE,
+ dt1.DATE_TIME,
+ li.DATE_TIME,
+ sum(li.C_CUSTKEY) sum_custkey
+FROM (SELECT '1996-01-02' AS DATE_TIME) dt1
+ LEFT JOIN
+ (SELECT third.LO_ORDERDATE,
+ third.LO_ORDERKEY,
+ third.C_NAME,
+ third.C_CUSTKEY,
+ third.DATE_TIME
+ FROM (SELECT dt2.LO_ORDERDATE,
+ dt2.LO_ORDERKEY,
+ dt2.C_NAME,
+ dt2.C_CUSTKEY,
+ dt4.DATE_TIME
+ FROM (SELECT LINEORDER.LO_ORDERDATE,
+ LINEORDER.LO_ORDERKEY,
+ CUSTOMER.C_NAME,
+ CUSTOMER.C_CUSTKEY,
+ dt.DATE_TIME
+ FROM "SSB"."LINEORDER" AS "LINEORDER"
+ INNER JOIN "SSB"."CUSTOMER" AS "CUSTOMER"
+ ON "LINEORDER"."LO_CUSTKEY" = "CUSTOMER"."C_CUSTKEY"
+ INNER JOIN
+ (SELECT '1995-03-29' AS DATE_TIME
+ UNION
+ SELECT '1995-04-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-03-29' AS DATE_TIME
+ UNION ALL
+ SELECT '1995-04-01' AS DATE_TIME
+ UNION ALL
+ SELECT '1996-01-02' AS DATE_TIME
+ UNION
+ SELECT CASE WHEN 1 = 1 THEN '1995-06-01' ELSE '1995-07-01' END AS DATE_TIME) dt
+ ON LINEORDER.LO_ORDERDATE = dt.DATE_TIME) dt2
+ INNER JOIN
+ (SELECT '1996-01-02' AS DATE_TIME) dt4 ON dt2.LO_ORDERDATE = dt4.DATE_TIME) third) li
+ ON dt1.DATE_TIME = li.LO_ORDERDATE
+GROUP BY li.C_NAME,
+ li.LO_ORDERDATE,
+ dt1.DATE_TIME,
+ li.DATE_TIME
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query19.sql b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query19.sql
new file mode 100644
index 0000000000..fe4e25e39c
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_scalar_subquery/query19.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- SQL not supported when there was RexCall in AggregateCall arguments.
+select sum(LINEORDER.LO_EXTENDEDPRICE * 3),
+ LINEORDER.LO_ORDERDATE,
+ dt.DATE_TIME
+from SSB.P_LINEORDER as LINEORDER
+ inner join (select 19931014 as DATE_TIME) dt
+ on LINEORDER.LO_ORDERDATE = dt.DATE_TIME
+group by LO_ORDERDATE, dt.DATE_TIME
\ No newline at end of file
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapValuesRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapValuesRel.java
index 352c108961..cf8188d0ed 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapValuesRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapValuesRel.java
@@ -18,17 +18,15 @@
package org.apache.kylin.query.relnode;
+import java.util.List;
import java.util.Set;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.RelCollationTraitDef;
-import org.apache.calcite.rel.RelDistributionTraitDef;
-import org.apache.calcite.rel.metadata.RelMdCollation;
-import org.apache.calcite.rel.metadata.RelMdDistribution;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexLiteral;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.query.util.ICutContextStrategy;
@@ -37,18 +35,15 @@ import com.google.common.collect.ImmutableList;
public class KapValuesRel extends OLAPValuesRel implements KapRel {
private Set<OLAPContext> subContexts = Sets.newHashSet();
- private KapValuesRel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples,
+ public KapValuesRel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples,
RelTraitSet traitSet) {
super(cluster, rowType, tuples, traitSet);
}
- public static KapValuesRel create(RelOptCluster cluster, final RelDataType rowType,
- final ImmutableList<ImmutableList<RexLiteral>> tuples) {
- final RelMetadataQuery mq = cluster.getMetadataQuery();
- final RelTraitSet traitSet = cluster.traitSetOf(OLAPRel.CONVENTION)
- .replaceIfs(RelCollationTraitDef.INSTANCE, () -> RelMdCollation.values(mq, rowType, tuples))
- .replaceIf(RelDistributionTraitDef.INSTANCE, () -> RelMdDistribution.values(rowType, tuples));
- return new KapValuesRel(cluster, rowType, tuples, traitSet);
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ Preconditions.checkArgument(inputs.isEmpty());
+ return new KapValuesRel(getCluster(), rowType, tuples, replaceTraitSet(getCluster(), rowType, tuples));
}
@Override
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPValuesRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPValuesRel.java
index 098e7712f8..55992e448f 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPValuesRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPValuesRel.java
@@ -38,6 +38,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexLiteral;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.TblColRef;
@@ -69,8 +70,8 @@ public class OLAPValuesRel extends Values implements OLAPRel {
@Override
public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- assert inputs.isEmpty();
- return create(getCluster(), rowType, tuples);
+ Preconditions.checkArgument(inputs.isEmpty());
+ return new OLAPValuesRel(getCluster(), rowType, tuples, replaceTraitSet(getCluster(), rowType, tuples));
}
@Override
@@ -119,4 +120,12 @@ public class OLAPValuesRel extends Values implements OLAPRel {
public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
return EnumerableValues.create(getCluster(), getRowType(), getTuples());
}
+
+ public static RelTraitSet replaceTraitSet(RelOptCluster cluster, RelDataType rowType, //
+ ImmutableList<ImmutableList<RexLiteral>> tuples) {
+ final RelMetadataQuery mq = cluster.getMetadataQuery();
+ return cluster.traitSetOf(OLAPRel.CONVENTION)
+ .replaceIfs(RelCollationTraitDef.INSTANCE, () -> RelMdCollation.values(mq, rowType, tuples))
+ .replaceIf(RelDistributionTraitDef.INSTANCE, () -> RelMdDistribution.values(rowType, tuples));
+ }
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KAPValuesRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KAPValuesRule.java
index f9db2c1ba9..de867da672 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KAPValuesRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KAPValuesRule.java
@@ -18,11 +18,15 @@
package io.kyligence.kap.query.optrule;
import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.convert.ConverterRule;
import org.apache.calcite.rel.core.Values;
-import org.apache.kylin.query.relnode.OLAPRel;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.kylin.query.relnode.KapValuesRel;
+import org.apache.kylin.query.relnode.OLAPRel;
+import org.apache.kylin.query.relnode.OLAPValuesRel;
public class KAPValuesRule extends ConverterRule {
public static final KAPValuesRule INSTANCE = new KAPValuesRule();
@@ -34,6 +38,9 @@ public class KAPValuesRule extends ConverterRule {
@Override
public RelNode convert(RelNode rel) {
Values values = (Values) rel;
- return KapValuesRel.create(values.getCluster(), values.getRowType(), values.getTuples());
+ RelOptCluster cluster = values.getCluster();
+ RelDataType rowType = values.getRowType();
+ RelTraitSet relTraits = OLAPValuesRel.replaceTraitSet(cluster, rowType, values.getTuples());
+ return new KapValuesRel(cluster, rowType, values.getTuples(), relTraits);
}
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/ScalarSubqueryJoinRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/ScalarSubqueryJoinRule.java
new file mode 100644
index 0000000000..e4b335334d
--- /dev/null
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/ScalarSubqueryJoinRule.java
@@ -0,0 +1,799 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.kyligence.kap.query.optrule;
+
+import java.math.BigDecimal;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlSplittableAggFunction;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.mapping.Mapping;
+import org.apache.calcite.util.mapping.MappingType;
+import org.apache.calcite.util.mapping.Mappings;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.query.relnode.KapAggregateRel;
+import org.apache.kylin.query.relnode.KapFilterRel;
+import org.apache.kylin.query.relnode.KapJoinRel;
+import org.apache.kylin.query.relnode.KapNonEquiJoinRel;
+import org.apache.kylin.query.relnode.KapProjectRel;
+import org.apache.kylin.query.relnode.KapValuesRel;
+
+public class ScalarSubqueryJoinRule extends RelOptRule {
+
+ // JOIN_PREDICATE from guava's Predicate,
+
+ public static final ScalarSubqueryJoinRule AGG_JOIN = new ScalarSubqueryJoinRule(//
+ operand(KapAggregateRel.class, //
+ operand(Join.class, //
+ null, j -> j instanceof KapJoinRel || j instanceof KapNonEquiJoinRel, any())),
+ RelFactories.LOGICAL_BUILDER, "ScalarSubqueryJoinRule:AGG_JOIN");
+
+ public static final ScalarSubqueryJoinRule AGG_PRJ_JOIN = new ScalarSubqueryJoinRule(//
+ operand(KapAggregateRel.class, //
+ operand(KapProjectRel.class, //
+ operand(Join.class, //
+ null, j -> j instanceof KapJoinRel || j instanceof KapNonEquiJoinRel, any()))),
+ RelFactories.LOGICAL_BUILDER, "ScalarSubqueryJoinRule:AGG_PRJ_JOIN");
+
+ public static final ScalarSubqueryJoinRule AGG_PRJ_FLT_JOIN = new ScalarSubqueryJoinRule(//
+ operand(KapAggregateRel.class, //
+ operand(KapProjectRel.class, //
+ operand(KapFilterRel.class, //
+ operand(Join.class, //
+ null, //
+ j -> j instanceof KapJoinRel || j instanceof KapNonEquiJoinRel, any())))), //
+ RelFactories.LOGICAL_BUILDER, "ScalarSubqueryJoinRule:AGG_PRJ_FLT_JOIN");
+
+ public ScalarSubqueryJoinRule(RelOptRuleOperand operand, RelBuilderFactory relBuilderFactory, String description) {
+ super(operand, relBuilderFactory, description);
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ Join join = call.rel(call.rels.length - 1);
+ switch (join.getJoinType()) {
+ case INNER:
+ case LEFT:
+ break;
+ default:
+ return false;
+ }
+
+ KapAggregateRel aggregate = call.rel(0);
+ if (!aggregate.isSimpleGroupType() || aggregate.getAggCallList().isEmpty()) {
+ return false;
+ }
+
+ // If any aggregate functions do not support splitting, bail outer
+ // If any aggregate call has a filter or is distinct, bail out.
+ // To-do: Count-distinct is currently not supported,
+ // but this can be achieved in case of scalar-subquery with distinct values.
+ if (aggregate.getAggCallList().stream().anyMatch(a -> a.hasFilter() //
+ || a.isDistinct() //
+ || Objects.isNull(a.getAggregation().unwrap(SqlSplittableAggFunction.class)))) {
+ return false;
+ }
+
+ return !(call.rel(1) instanceof KapProjectRel) || canApplyRule(call.rel(1));
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ Transposer transposer = new Transposer(call);
+ if (!transposer.canTranspose()) {
+ return;
+ }
+
+ RelNode relNode = transposer.getTransposedRel();
+ call.transformTo(relNode);
+ }
+
+ private boolean canApplyRule(KapProjectRel project) {
+ if (project.getProjects().stream().anyMatch(RexCall.class::isInstance)) {
+ // to-do: maybe we should support this.
+ return false;
+ }
+ final ImmutableBitSet.Builder builder = ImmutableBitSet.builder();
+ project.getProjects().forEach(p -> builder.addAll(RelOptUtil.InputFinder.bits(p)));
+ // Avoid re-entry of rule.
+ return project.getProjects().size() <= builder.build().cardinality();
+ }
+
+ private <E> SqlSplittableAggFunction.Registry<E> createRegistry(final List<E> list) {
+ return e -> {
+ int i = list.indexOf(e);
+ if (i < 0) {
+ i = list.size();
+ list.add(e);
+ }
+ return i;
+ };
+ }
+
+ private class Transposer {
+ // base variables
+ private final RelOptRuleCall call;
+ private final AggregateUnit aggUnit;
+ private final Join join;
+
+ // join sides
+ private final LeftSide left;
+ private final RightSide right;
+
+ // maintainer
+ public Transposer(RelOptRuleCall ruleCall) {
+ // call -> "aggunit, join"
+ call = ruleCall;
+ aggUnit = createAggUnit(ruleCall);
+ join = ruleCall.rel(ruleCall.rels.length - 1);
+
+ // join sides
+ RelMetadataQuery mq = call.getMetadataQuery();
+ ImmutableBitSet joinCondSet = RelOptUtil.InputFinder.bits(join.getCondition());
+ ImmutableBitSet aggUnitJoinSet = aggUnit.getUnitSet().union(joinCondSet);
+ left = new LeftSide(join.getLeft(), join.getInput(0), mq, aggUnitJoinSet);
+ right = new RightSide(left, join.getRight(), join.getInput(1), mq, aggUnitJoinSet);
+ }
+
+ public boolean canTranspose() {
+ if (left.hasRelValues() && right.isAggregable()) {
+ return true;
+ }
+
+ return right.hasRelValues() && left.isAggregable();
+ }
+
+ public RelNode getTransposedRel() {
+ // builders
+ final RelBuilder relBuilder = call.builder();
+ final RexBuilder rexBuilder = aggUnit.getRexBuilder();
+
+ // below aggregate
+ left.modifyAggregate(aggUnit, relBuilder, rexBuilder);
+ right.modifyAggregate(aggUnit, relBuilder, rexBuilder);
+
+ // aggunit-join mapping
+ final Mapping aggUnitJoinMapping = getAggUnitJoinMapping();
+
+ // create new join
+ final RexNode joinCond = RexUtil.apply(aggUnitJoinMapping, join.getCondition());
+ relBuilder.push(left.getNewInput()).push(right.getNewInput()).join(join.getJoinType(), joinCond);
+
+ if (aggUnit instanceof AggregateProjectFilter) {
+ // create new filter
+ // To-do: maybe we could also push down the relative filter.
+ final RexNode filterCond = RexUtil.apply(aggUnitJoinMapping, //
+ ((AggregateProjectFilter) aggUnit).getFilterCond());
+ relBuilder.filter(filterCond);
+ }
+
+ // agg-project mapping
+ final Mapping projectMapping = getProjectMapping(relBuilder, aggUnitJoinMapping);
+
+ // aggregate above to sum up the sub-totals
+ final List<RexNode> projectList = //
+ Mappings.apply(projectMapping, //
+ Lists.newArrayList(rexBuilder.identityProjects(relBuilder.peek().getRowType())));
+
+ final List<AggregateCall> aggCallList = Lists.newArrayList();
+ aggregateAbove(projectList, aggCallList, relBuilder, rexBuilder);
+
+ // create new project
+ relBuilder.project(projectList);
+
+ // above aggregate
+ // To-do: maybe we could convert aggregate into projects when inner-join.
+ final RelBuilder.GroupKey groupKey = //
+ relBuilder.groupKey(Mappings.apply(projectMapping, //
+ Mappings.apply(aggUnitJoinMapping, aggUnit.getGroupSet())), //
+ Mappings.apply2(projectMapping, //
+ Mappings.apply2(aggUnitJoinMapping, aggUnit.getGroupSets())));
+ relBuilder.aggregate(groupKey, aggCallList);
+
+ return relBuilder.build();
+ }
+
+ private Mapping getProjectMapping(final RelBuilder relBuilder, final Mapping aggMapping) {
+ final List<Integer> fieldList = //
+ IntStream.range(0, relBuilder.peek().getRowType().getFieldList().size()) //
+ .boxed().collect(Collectors.toList());
+ final List<Integer> groupList = //
+ aggUnit.getGroupList().stream().map(aggMapping::getTarget).collect(Collectors.toList());
+
+ // [i0, i1, i2, i3, i4] -> [i1, i3, i0, i2, i4]
+ final Mapping projectMapping = //
+ Mappings.create(MappingType.BIJECTION, fieldList.size(), fieldList.size());
+
+ Ord.zip(fieldList).forEach(o -> projectMapping.set(o.i, o.e));
+ Ord.zip(groupList).forEach(o -> projectMapping.set(o.e, o.i));
+
+ return projectMapping;
+ }
+
+ private AggregateUnit createAggUnit(RelOptRuleCall call) {
+ if (call.rels.length > 3) {
+ return new AggregateProjectFilter(call.rel(0), call.rel(1), call.rel(2));
+ }
+ if (call.rels.length > 2) {
+ return new AggregateProject(call.rel(0), call.rel(1));
+ }
+ return new AggregateUnit(call.rel(0));
+ }
+
+ private void aggregateAbove(final List<RexNode> projectList, //
+ final List<AggregateCall> aggCallList, //
+ final RelBuilder relBuilder, //
+ final RexBuilder rexBuilder) {
+ final int newLeftWidth = left.getNewInputFieldCount();
+ final int groupIndicatorCount = aggUnit.getGroupIndicatorCount();
+ final SqlSplittableAggFunction.Registry<RexNode> projectRegistry = createRegistry(projectList);
+ Ord.zip(aggUnit.getAggCallList()).forEach(aggCallOrd -> {
+ // No need to care about args' mapping.
+ AggregateCall aggCall = aggCallOrd.e;
+ SqlAggFunction aggFunc = aggCall.getAggregation();
+ SqlSplittableAggFunction splitAggFunc = Preconditions
+ .checkNotNull(aggFunc.unwrap(SqlSplittableAggFunction.class));
+ Integer lst = left.getAggOrdinal(aggCallOrd.i);
+ Integer rst = right.getAggOrdinal(aggCallOrd.i);
+ final AggregateCall newAggCall = //
+ splitAggFunc.topSplit(rexBuilder, projectRegistry, groupIndicatorCount, //
+ relBuilder.peek().getRowType(), aggCall, //
+ Objects.isNull(lst) ? -1 : lst, //
+ Objects.isNull(rst) ? -1 : rst + newLeftWidth);
+
+ if (aggCall.getAggregation() == SqlStdOperatorTable.COUNT //
+ && newAggCall.getAggregation() == SqlStdOperatorTable.SUM0) {
+ aboveCountSum0(rexBuilder, aggCall, newAggCall, projectList);
+ }
+
+ aggCallList.add(newAggCall);
+ });
+ }
+
+ private void aboveCountSum0(final RexBuilder rexBuilder, //
+ AggregateCall aggCall, //
+ AggregateCall newAggCall, //
+ final List<RexNode> projectList) {
+ // COUNT(*), COUNT(1), COUNT(COL), COUNT(COL1, COL2, ...)
+ final boolean nullAsOne = aggCall.getArgList().isEmpty();
+ newAggCall.getArgList().forEach(i -> {
+ // old project node
+ RexNode p = projectList.get(i);
+ // when-then-else
+ List<RexNode> wte = Lists.newLinkedList();
+ wte.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, p));
+ if (nullAsOne) {
+ wte.add(rexBuilder.makeLiteral(BigDecimal.ONE, p.getType(), true));
+ } else {
+ wte.add(rexBuilder.makeZeroLiteral(p.getType()));
+ }
+
+ wte.add(p);
+ // new project node
+ RexNode np = rexBuilder.makeCall(p.getType(), SqlStdOperatorTable.CASE, wte);
+ projectList.set(i, np);
+ });
+ }
+
+ private Mapping getAggUnitJoinMapping() {
+ final Map<Integer, Integer> map = Maps.newHashMap();
+ map.putAll(left.getAggUnitJoinMap());
+ map.putAll(right.getAggUnitJoinMap());
+ final int sourceCount = join.getRowType().getFieldCount();
+ final int targetCount = left.getNewInputFieldCount() + right.getNewInputFieldCount();
+ return (Mapping) Mappings.target(map::get, sourceCount, targetCount);
+ }
+
+ } // end of Transposer
+
+ private static class AggregateUnit {
+ // base variables
+ protected final KapAggregateRel aggregate;
+
+ // immediate variables
+ private RexBuilder rexBuilder;
+
+ public AggregateUnit(KapAggregateRel aggregate) {
+ this.aggregate = aggregate;
+ }
+
+ public RexBuilder getRexBuilder() {
+ if (Objects.isNull(rexBuilder)) {
+ rexBuilder = aggregate.getCluster().getRexBuilder();
+ }
+ return rexBuilder;
+ }
+
+ public ImmutableBitSet getUnitSet() {
+ return getGroupSet();
+ }
+
+ public ImmutableBitSet getGroupSet() {
+ return aggregate.getGroupSet();
+ }
+
+ public ImmutableList<ImmutableBitSet> getGroupSets() {
+ return ImmutableList.<ImmutableBitSet> builder().addAll(aggregate.groupSets).build();
+ }
+
+ public int getGroupCount() {
+ return aggregate.getGroupCount();
+ }
+
+ public int getGroupIndicatorCount() {
+ return getGroupCount() + aggregate.getIndicatorCount();
+ }
+
+ public List<AggregateCall> getAggCallList() {
+ return aggregate.getAggCallList();
+ }
+
+ public List<Integer> getGroupList() {
+ return aggregate.getGroupSet().asList();
+ }
+
+ } // end of AggregateUnit
+
+ private static class AggregateProject extends AggregateUnit {
+ // base variables
+ private final KapProjectRel project;
+ private final Mappings.TargetMapping targetMapping;
+
+ // immediate variables
+ private ImmutableBitSet groupSet;
+ private ImmutableList<ImmutableBitSet> groupSets;
+ private List<AggregateCall> aggCallList;
+
+ public AggregateProject(KapAggregateRel aggregate, KapProjectRel project) {
+ super(aggregate);
+ this.project = project;
+ this.targetMapping = createTargetMapping();
+ }
+
+ @Override
+ public ImmutableBitSet getGroupSet() {
+ if (Objects.isNull(groupSet)) {
+ groupSet = Mappings.apply((Mapping) targetMapping, aggregate.getGroupSet());
+ }
+ return groupSet;
+ }
+
+ @Override
+ public ImmutableList<ImmutableBitSet> getGroupSets() {
+ if (Objects.isNull(groupSets)) {
+ groupSets = ImmutableList.<ImmutableBitSet> builder() //
+ .addAll(Mappings.apply2((Mapping) targetMapping, aggregate.getGroupSets())) //
+ .build();
+ }
+ return groupSets;
+ }
+
+ @Override
+ public List<AggregateCall> getAggCallList() {
+ if (Objects.isNull(aggCallList)) {
+ aggCallList = aggregate.getAggCallList().stream() //
+ .map(a -> a.transform(targetMapping)) //
+ .collect(Collectors.collectingAndThen(Collectors.toList(), //
+ Collections::unmodifiableList));
+ }
+ return aggCallList;
+ }
+
+ @Override
+ public List<Integer> getGroupList() {
+ return super.getGroupList().stream().map(targetMapping::getTarget).collect(Collectors.toList());
+ }
+
+ private Mappings.TargetMapping createTargetMapping() {
+ if (Objects.isNull(project.getMapping())) {
+ return Mappings.createIdentity(project.getRowType().getFieldCount());
+ }
+ return project.getMapping().inverse();
+ }
+
+ } // end of AggregateProject
+
+ private static class AggregateProjectFilter extends AggregateProject {
+
+ private final KapFilterRel filter;
+
+ public AggregateProjectFilter(KapAggregateRel aggregate, KapProjectRel project, KapFilterRel filter) {
+ super(aggregate, project);
+ this.filter = filter;
+ }
+
+ @Override
+ public ImmutableBitSet getUnitSet() {
+ ImmutableBitSet filterSet = RelOptUtil.InputFinder.bits(filter.getCondition());
+ return getGroupSet().union(filterSet);
+ }
+
+ public RexNode getFilterCond() {
+ return filter.getCondition();
+ }
+
+ } // end of AggregateProjectFilter
+
+ private abstract class JoinSide {
+ // base variables
+ private final boolean isRelValues;
+ private final boolean hasRelValues;
+ private final RelNode input;
+ private final ImmutableBitSet aggUnitJoinSet;
+
+ // util variables
+ protected ImmutableBitSet fieldSet;
+ protected ImmutableBitSet sideAggUnitJoinSet;
+ protected ImmutableBitSet belowAggGroupSet;
+
+ // immediate variables
+ private Map<Integer, Integer> aggOrdinalMap;
+ private List<AggregateCall> belowAggCallList;
+ private SqlSplittableAggFunction.Registry<AggregateCall> belowAggCallRegistry;
+
+ private RelNode newInput;
+
+ private Map<Integer, Integer> aggUnitJoinMap;
+
+ public JoinSide(RelNode relNode, RelNode input, ImmutableBitSet aggUnitJoinSet) {
+ this.isRelValues = isRelValues(relNode);
+ this.hasRelValues = hasRelValues(relNode);
+ this.input = input;
+ this.aggUnitJoinSet = aggUnitJoinSet;
+ }
+
+ public boolean hasRelValues() {
+ return hasRelValues;
+ }
+
+ public boolean isRelValues() {
+ return isRelValues;
+ }
+
+ public abstract boolean isAggregable();
+
+ public RelNode getNewInput() {
+ return newInput;
+ }
+
+ public Integer getAggOrdinal(int i) {
+ return getAggOrdinalMap().get(i);
+ }
+
+ public Map<Integer, Integer> getAggUnitJoinMap() {
+ if (Objects.isNull(aggUnitJoinMap)) {
+ final int belowOffset = getBelowOffset();
+ final Map<Integer, Integer> map = Maps.newHashMap();
+ Ord.zip(getSideAggUnitJoinSet()).forEach(o -> map.put(o.e, belowOffset + o.i));
+ aggUnitJoinMap = map;
+ }
+ return aggUnitJoinMap;
+ }
+
+ public void modifyAggregate(AggregateUnit aggUnit, RelBuilder relBuilder, RexBuilder rexBuilder) {
+ if (isRelValues()) {
+ newInput = convertSingleton(aggUnit, relBuilder, rexBuilder);
+ return;
+ }
+
+ if (isAggregable()) {
+ newInput = convertSplit(aggUnit, relBuilder, rexBuilder);
+ return;
+ }
+
+ newInput = convertSingleton(aggUnit, relBuilder, rexBuilder);
+ }
+
+ protected final boolean hasRelValues(RelNode node) {
+ if (node instanceof HepRelVertex) {
+ RelNode current = ((HepRelVertex) node).getCurrentRel();
+ if (current instanceof Join) {
+ final Join join = (Join) current;
+ return isRelValues(join.getLeft()) || isRelValues(join.getRight());
+ }
+ return isRelValues(node);
+ }
+ return false;
+ }
+
+ protected final boolean isRelValues(RelNode node) {
+ if (node instanceof HepRelVertex) {
+ RelNode current = ((HepRelVertex) node).getCurrentRel();
+ if (current instanceof KapValuesRel) {
+ return true;
+ }
+
+ if (current.getInputs().isEmpty()) {
+ return false;
+ }
+
+ return current.getInputs().stream().allMatch(this::isRelValues);
+ }
+ return false;
+ }
+
+ protected final boolean isAggregable(RelNode input, RelMetadataQuery mq) {
+ // No need to do aggregation. There is nothing to be gained by this rule.
+ Boolean unique = mq.areColumnsUnique(input, getBelowAggGroupSet());
+ return Objects.isNull(unique) || !unique;
+ }
+
+ protected int getInputFieldCount() {
+ return input.getRowType().getFieldCount();
+ }
+
+ protected int getNewInputFieldCount() {
+ return Preconditions.checkNotNull(newInput).getRowType().getFieldCount();
+ }
+
+ protected abstract int getOffset();
+
+ protected abstract int getBelowOffset();
+
+ protected abstract Mappings.TargetMapping getTargetMapping();
+
+ private Map<Integer, Integer> getAggOrdinalMap() {
+ if (Objects.isNull(aggOrdinalMap)) {
+ aggOrdinalMap = Maps.newHashMap();
+ }
+ return aggOrdinalMap;
+ }
+
+ private void registryAggCall(int i, int offset, AggregateCall aggCall) {
+ getAggOrdinalMap().put(i, offset + registry(aggCall));
+ }
+
+ private void registryOther(int i, int ordinal) {
+ getAggOrdinalMap().put(i, ordinal);
+ }
+
+ private ImmutableBitSet getFieldSet() {
+ if (Objects.isNull(fieldSet)) {
+ int offset = getOffset();
+ fieldSet = ImmutableBitSet.range(offset, offset + getInputFieldCount());
+ }
+ return fieldSet;
+ }
+
+ private ImmutableBitSet getBelowAggGroupSet() {
+ if (Objects.isNull(belowAggGroupSet)) {
+ int offset = getOffset();
+ belowAggGroupSet = getSideAggUnitJoinSet().shift(-offset);
+ }
+ return belowAggGroupSet;
+ }
+
+ private ImmutableBitSet getSideAggUnitJoinSet() {
+ if (Objects.isNull(sideAggUnitJoinSet)) {
+ ImmutableBitSet fieldSet0 = getFieldSet();
+ sideAggUnitJoinSet = Preconditions.checkNotNull(aggUnitJoinSet).intersect(fieldSet0);
+ }
+ return sideAggUnitJoinSet;
+ }
+
+ private RelNode convertSplit(AggregateUnit aggUnit, RelBuilder relBuilder, RexBuilder rexBuilder) {
+ final ImmutableBitSet fields = getFieldSet();
+ final int oldGroupSetCount = aggUnit.getGroupCount();
+ final int newGroupSetCount = getBelowAggGroupSet().cardinality();
+ Ord.zip(aggUnit.getAggCallList()).forEach(aggCallOrd -> {
+ AggregateCall aggCall = aggCallOrd.e;
+ SqlAggFunction aggFunc = aggCall.getAggregation();
+ SqlSplittableAggFunction splitAggFunc = Preconditions
+ .checkNotNull(aggFunc.unwrap(SqlSplittableAggFunction.class));
+ ImmutableBitSet aggArgSet = ImmutableBitSet.of(aggCall.getArgList());
+ final AggregateCall newAggCall;
+ if (fields.contains(aggArgSet)) {
+ // convert split
+ AggregateCall splitAggCall = splitAggFunc.split(aggCall, getTargetMapping());
+ newAggCall = splitAggCall.adaptTo(input, splitAggCall.getArgList(), splitAggCall.filterArg, //
+ oldGroupSetCount, newGroupSetCount);
+ } else {
+ newAggCall = splitOther(splitAggFunc, rexBuilder, aggCall, fields, aggArgSet);
+ }
+
+ if (Objects.isNull(newAggCall)) {
+ return;
+ }
+ registryAggCall(aggCallOrd.i, newGroupSetCount, newAggCall);
+ });
+
+ return relBuilder.push(input) //
+ .aggregate(relBuilder.groupKey(belowAggGroupSet, null), //
+ Preconditions.checkNotNull(belowAggCallList)) //
+ .build();
+ }
+
+ private AggregateCall splitOther(SqlSplittableAggFunction splitAggFunc, //
+ RexBuilder rexBuilder, //
+ AggregateCall aggCall, //
+ ImmutableBitSet fields, //
+ ImmutableBitSet args) {
+ // Thinking...aggCall not transformed?
+ AggregateCall other = splitAggFunc.other(rexBuilder.getTypeFactory(), aggCall);
+ if (Objects.isNull(other)) {
+ return null;
+ }
+
+ ImmutableBitSet newArgSet = Mappings.apply((Mapping) getTargetMapping(), args.intersect(fields));
+ return AggregateCall.create(other.getAggregation(), other.isDistinct(), //
+ other.isApproximate(), newArgSet.asList(), //
+ other.filterArg, other.getType(), other.getName());
+
+ }
+
+ private RelNode convertSingleton(AggregateUnit aggUnit, RelBuilder relBuilder, RexBuilder rexBuilder) {
+ relBuilder.push(input);
+ final ImmutableBitSet fieldSet0 = getFieldSet();
+ final List<RexNode> projectList = Lists.newArrayList();
+ getBelowAggGroupSet().forEach(i -> projectList.add(relBuilder.field(i)));
+ Ord.zip(aggUnit.getAggCallList()).forEach(aggCallOrd -> {
+ AggregateCall aggCall = aggCallOrd.e;
+ SqlAggFunction aggFunc = aggCall.getAggregation();
+ SqlSplittableAggFunction splitAggFunc = Preconditions
+ .checkNotNull(aggFunc.unwrap(SqlSplittableAggFunction.class));
+ if (aggCall.getArgList().isEmpty()) {
+ return;
+ }
+ ImmutableBitSet aggArgSet = ImmutableBitSet.of(aggCall.getArgList());
+ if (!fieldSet0.contains(aggArgSet)) {
+ return;
+ }
+
+ // convert singleton
+ RexNode singleton = splitAggFunc.singleton(rexBuilder, input.getRowType(), //
+ aggCall.transform(getTargetMapping()));
+ if (singleton instanceof RexInputRef) {
+ registryOther(aggCallOrd.i, ((RexInputRef) singleton).getIndex());
+ return;
+ }
+ int ordinal = projectList.size();
+ projectList.add(singleton);
+ registryOther(aggCallOrd.i, ordinal);
+ });
+
+ relBuilder.project(projectList);
+
+ return relBuilder.build();
+ }
+
+ private int registry(AggregateCall aggCall) {
+ if (Objects.isNull(belowAggCallRegistry)) {
+ if (Objects.isNull(belowAggCallList)) {
+ belowAggCallList = Lists.newArrayList();
+ }
+ belowAggCallRegistry = createRegistry(belowAggCallList);
+ }
+ return belowAggCallRegistry.register(aggCall);
+ }
+
+ } // end of side
+
+ private class LeftSide extends JoinSide {
+
+ private final boolean isAggregable;
+
+ private final Mappings.TargetMapping targetMapping;
+
+ public LeftSide(RelNode relNode, RelNode input, RelMetadataQuery mq, ImmutableBitSet aggUnitJoinSet) {
+ super(relNode, input, aggUnitJoinSet);
+ this.isAggregable = isAggregable(input, mq);
+ this.targetMapping = createTargetMapping();
+ }
+
+ @Override
+ public boolean isAggregable() {
+ return isAggregable;
+ }
+
+ @Override
+ protected int getOffset() {
+ return 0;
+ }
+
+ @Override
+ protected int getBelowOffset() {
+ return 0;
+ }
+
+ @Override
+ protected Mappings.TargetMapping getTargetMapping() {
+ return targetMapping;
+ }
+
+ private Mappings.TargetMapping createTargetMapping() {
+ int fieldCount = getInputFieldCount();
+ return Mappings.createIdentity(fieldCount);
+ }
+
+ } // end of LeftSide
+
+ private class RightSide extends JoinSide {
+
+ private final LeftSide left;
+
+ private final boolean isAggregable;
+
+ private final Mappings.TargetMapping targetMapping;
+
+ public RightSide(LeftSide left, //
+ RelNode relNode, RelNode input, //
+ RelMetadataQuery mq, ImmutableBitSet aggUnitJoinSet) {
+ super(relNode, input, aggUnitJoinSet);
+ this.left = left;
+ this.isAggregable = isAggregable(input, mq);
+ this.targetMapping = createTargetMapping();
+ }
+
+ @Override
+ public boolean isAggregable() {
+ return isAggregable;
+ }
+
+ @Override
+ protected int getOffset() {
+ return left.getInputFieldCount();
+ }
+
+ @Override
+ protected int getBelowOffset() {
+ return left.getNewInputFieldCount();
+ }
+
+ @Override
+ protected Mappings.TargetMapping getTargetMapping() {
+ return targetMapping;
+ }
+
+ private Mappings.TargetMapping createTargetMapping() {
+ int offset = getOffset();
+ int fieldCount = getInputFieldCount();
+ return Mappings.createShiftMapping(fieldCount + offset, 0, offset, fieldCount);
+ }
+
+ } // end of RightSide
+
+}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index 51e4e4261d..dc0572b6f8 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -267,6 +267,10 @@ public class QueryExec {
postOptRules.addAll(HepUtils.AggPushDownRules);
}
+ if(kylinConfig.isScalarSubqueryJoinEnabled()) {
+ postOptRules.addAll(HepUtils.ScalarSubqueryJoinRules);
+ }
+
if (kylinConfig.isOptimizedSumCastDoubleRuleEnabled()) {
postOptRules.addAll(HepUtils.SumCastDoubleRules);
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/HepUtils.java b/src/query/src/main/java/org/apache/kylin/query/util/HepUtils.java
index bea632f49f..92be45067d 100644
--- a/src/query/src/main/java/org/apache/kylin/query/util/HepUtils.java
+++ b/src/query/src/main/java/org/apache/kylin/query/util/HepUtils.java
@@ -25,11 +25,13 @@ import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rel.rules.FilterMergeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
import org.apache.calcite.rel.rules.ProjectRemoveRule;
-
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import io.kyligence.kap.query.optrule.AggregateProjectReduceRule;
import io.kyligence.kap.query.optrule.CountDistinctCaseWhenFunctionRule;
import io.kyligence.kap.query.optrule.FilterJoinConditionMergeRule;
import io.kyligence.kap.query.optrule.FilterSimplifyRule;
@@ -49,6 +51,7 @@ import io.kyligence.kap.query.optrule.KapProjectMergeRule;
import io.kyligence.kap.query.optrule.KapProjectRule;
import io.kyligence.kap.query.optrule.KapSumCastTransposeRule;
import io.kyligence.kap.query.optrule.KapSumTransCastToThenRule;
+import io.kyligence.kap.query.optrule.ScalarSubqueryJoinRule;
import io.kyligence.kap.query.optrule.SumBasicOperatorRule;
import io.kyligence.kap.query.optrule.SumCaseWhenFunctionRule;
import io.kyligence.kap.query.optrule.SumConstantConvertRule;
@@ -97,6 +100,21 @@ public class HepUtils {
KapJoinRule.INSTANCE
);
+ public static final ImmutableList<RelOptRule> ScalarSubqueryJoinRules = ImmutableList.of(
+ // base rules
+ KapAggregateRule.INSTANCE, //
+ KapProjectRule.INSTANCE, //
+ KapJoinRule.INSTANCE, //
+ // relative rules
+ ProjectMergeRule.INSTANCE, //
+ AggregateProjectMergeRule.INSTANCE, //
+ AggregateProjectReduceRule.INSTANCE, //
+ // target rules
+ ScalarSubqueryJoinRule.AGG_JOIN, //
+ ScalarSubqueryJoinRule.AGG_PRJ_JOIN, //
+ ScalarSubqueryJoinRule.AGG_PRJ_FLT_JOIN
+ );
+
public static final ImmutableList<RelOptRule> CountDistinctExprRules = ImmutableList.of(
CountDistinctCaseWhenFunctionRule.INSTANCE,
KapProjectRule.INSTANCE,
(kylin) 06/35: KYLIN-5798 Add user-table access cache for table-loading
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4d3418c237d4a839a85573712d0111d81086a477
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Wed Aug 30 14:11:32 2023 +0800
KYLIN-5798 Add user-table access cache for table-loading
---
.../org/apache/kylin/common/KylinConfigBase.java | 13 +++++++++++
.../apache/kylin/common/KylinConfigBaseTest.java | 22 ++++++++++++++++++
.../apache/kylin/rest/service/SparkDDLTest.java | 26 ++++++++++------------
.../spark/source/NSparkMetadataExplorer.java | 22 +++++++++++++++---
.../spark/source/NSparkMetadataExplorerTest.java | 14 +++++++++++-
5 files changed, 79 insertions(+), 18 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3de55bf84e..8b9a0308f7 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2800,6 +2800,19 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.source.hive.table-access-filter-enabled", FALSE));
}
+ public boolean getTableAccessCacheEnable() {
+ return Boolean.parseBoolean(getOptional("kylin.source.hive.table-access-cache-enabled", TRUE));
+ }
+
+ public long getTableAccessCacheSize() {
+ return Long.parseLong(getOptional("kylin.source.hive.table-access-cache-size", ONE_HUNDRED_THOUSAND));
+ }
+
+ public long getTableAccessCacheTTL() {
+ return TimeUtil.timeStringAs(getOptional("kylin.source.hive.table-access-cache-ttl", "7d"),
+ TimeUnit.MINUTES);
+ }
+
public String[] getHiveDatabases() {
return getOptionalStringArray("kylin.source.hive.databases", new String[0]);
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 6be4411004..59e4a3fc9b 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -36,6 +36,7 @@
package org.apache.kylin.common;
+import static org.apache.kylin.common.KylinConfigBase.FALSE;
import static org.apache.kylin.common.KylinConfigBase.PATH_DELIMITER;
import static org.apache.kylin.common.KylinConfigBase.WRITING_CLUSTER_WORKING_DIR;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_ENABLE_KEY;
@@ -1499,6 +1500,27 @@ class KylinConfigBaseTest {
config.setProperty("kylin.metadata.audit-log.max-size", "3000000");
assertEquals(3000000, config.getMetadataAuditLogMaxSize());
}
+
+ @Test
+ void testGetTableAccessCache() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+
+ assertTrue(config.getTableAccessCacheEnable());
+ config.setProperty("kylin.source.hive.table-access-cache-enabled", FALSE);
+ assertFalse(config.getTableAccessCacheEnable());
+
+ assertEquals(100000, config.getTableAccessCacheSize());
+ config.setProperty("kylin.source.hive.table-access-cache-size", "200000");
+ assertEquals(200000, config.getTableAccessCacheSize());
+
+ assertEquals(10080, config.getTableAccessCacheTTL());
+ config.setProperty("kylin.source.hive.table-access-cache-ttl", "1m");
+ assertEquals(1, config.getTableAccessCacheTTL());
+ config.setProperty("kylin.source.hive.table-access-cache-ttl", "1h");
+ assertEquals(60, config.getTableAccessCacheTTL());
+ config.setProperty("kylin.source.hive.table-access-cache-ttl", "1d");
+ assertEquals(1440, config.getTableAccessCacheTTL());
+ }
}
class EnvironmentUpdateUtils {
diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
index cf23d159aa..dc293e2b0c 100644
--- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
+++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -41,36 +42,27 @@ import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.ViewRequest;
import org.apache.kylin.rest.response.LoadTableResponse;
import org.apache.kylin.rest.response.LogicalViewResponse;
-
import org.apache.spark.sql.LogicalViewLoader;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.common.SparkDDLTestUtils;
-
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
public class SparkDDLTest extends NLocalFileMetadataTestCase {
- @Autowired
- private final SparkDDLService ddlService = Mockito.spy(new SparkDDLService());
- @Autowired
- private final TableExtService tableExtService = Mockito.spy(new TableExtService());
- @Autowired
- private final TableService tableService = Mockito.spy(new TableService());
- @Autowired
- private final IUserGroupService userGroupService = Mockito.spy(NUserGroupService.class);
- private final Integer LOGICAL_VIEW_CATCHUP_INTERVAL = 3;
+ private SparkDDLService ddlService;
+ private TableExtService tableExtService;
+ private TableService tableService;
+ private IUserGroupService userGroupService;
+ private Integer LOGICAL_VIEW_CATCHUP_INTERVAL = 3;
// Hive View
private static final String CREATEVIEW_SQL1 =
@@ -122,6 +114,12 @@ public class SparkDDLTest extends NLocalFileMetadataTestCase {
@Before
public void setup() {
createTestMetadata();
+
+ ddlService = Mockito.spy(new SparkDDLService());
+ tableExtService = Mockito.spy(new TableExtService());
+ tableService = Mockito.spy(new TableService());
+ userGroupService = Mockito.spy(NUserGroupService.class);
+
Authentication authentication = new TestingAuthenticationToken("ADMIN",
"ADMIN", Constant.ROLE_ADMIN);
SecurityContextHolder.getContext().setAuthentication(authentication);
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
index 3ade78d63a..6f3b1cf5ed 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -41,6 +43,8 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.cache.Cache;
+import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ISourceAware;
@@ -67,6 +71,9 @@ import lombok.val;
public class NSparkMetadataExplorer implements ISourceMetadataExplorer, ISampleDataDeployer, Serializable {
private static final Logger logger = LoggerFactory.getLogger(NSparkMetadataExplorer.class);
+ private static Cache<String, Boolean> tableAccessCache = CacheBuilder.newBuilder()
+ .maximumSize(KylinConfig.getInstanceFromEnv().getTableAccessCacheSize())
+ .expireAfterWrite(KylinConfig.getInstanceFromEnv().getTableAccessCacheTTL(), TimeUnit.MINUTES).build();
public static String generateCreateSchemaSql(String schemaName) {
return String.format(Locale.ROOT, "CREATE DATABASE IF NOT EXISTS %s", schemaName);
@@ -122,7 +129,6 @@ public class NSparkMetadataExplorer implements ISourceMetadataExplorer, ISampleD
public List<String> listTables(String database) throws Exception {
val ugi = UserGroupInformation.getCurrentUser();
val config = KylinConfig.getInstanceFromEnv();
- val spark = SparderEnv.getSparkSession();
List<String> tables = Lists.newArrayList();
try {
@@ -136,9 +142,10 @@ public class NSparkMetadataExplorer implements ISourceMetadataExplorer, ISampleD
if (config.getTableAccessFilterEnable() && config.getKerberosProjectLevelEnable()
&& UserGroupInformation.isSecurityEnabled()) {
List<String> accessTables = Lists.newArrayList();
+ boolean cacheEnabled = config.getTableAccessCacheEnable();
for (String table : tables) {
val tableName = database + "." + table;
- if (checkTableAccess(tableName)) {
+ if (checkTableWithCache(cacheEnabled, ugi.getUserName(), tableName)) {
accessTables.add(table);
}
}
@@ -151,6 +158,15 @@ public class NSparkMetadataExplorer implements ISourceMetadataExplorer, ISampleD
return tables;
}
+ public boolean checkTableWithCache(boolean cacheEnabled, String user, String tableName) throws ExecutionException {
+ if (cacheEnabled) {
+ String cacheKey = user + "#tableAccess#" + tableName;
+ return tableAccessCache.get(cacheKey, () -> checkTableAccess(tableName));
+ } else {
+ return checkTableAccess(tableName);
+ }
+ }
+
public boolean checkTableAccess(String tableName) {
boolean isAccess = true;
try {
@@ -257,7 +273,7 @@ public class NSparkMetadataExplorer implements ISourceMetadataExplorer, ISampleD
.orElseGet(Collections::emptyList).stream().map(field -> field.name) //
.collect(Collectors.toSet());
int columnNumber = tableMeta.allColumns.size();
- List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);
+ List<ColumnDesc> columns = new ArrayList<>(columnNumber);
for (int i = 0; i < columnNumber; i++) {
NSparkTableMeta.SparkTableColumnMeta field = tableMeta.allColumns.get(i);
ColumnDesc cdesc = new ColumnDesc();
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorerTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorerTest.java
index fddfa930a5..4ed485541b 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorerTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorerTest.java
@@ -47,9 +47,21 @@ public class NSparkMetadataExplorerTest extends NLocalWithSparkSessionTest {
@Test
public void testListTables() throws Exception {
+ String testProject = "default";
+ String testTableName = "DEFAULT.TEST_KYLIN_FACT";
+
NSparkMetadataExplorer sparkMetadataExplorer = new NSparkMetadataExplorer();
- List<String> tables = sparkMetadataExplorer.listTables("");
+ NTableMetadataManager tableMgr = NTableMetadataManager.getInstance(getTestConfig(), testProject);
+ TableDesc fact = tableMgr.getTableDesc(testTableName);
+ sparkMetadataExplorer.createSampleTable(fact);
+
+ List<String> tables = sparkMetadataExplorer.listTables(testProject);
Assert.assertTrue(tables != null && tables.size() > 0);
+
+ // test tableAccessCache, use only under Project Kerberos
+ boolean access = sparkMetadataExplorer.checkTableWithCache(false, "testUser", testTableName);
+ boolean cacheAccess = sparkMetadataExplorer.checkTableWithCache(true, "testUser", testTableName);
+ Assert.assertEquals(access, cacheAccess);
}
@Test
(kylin) 01/35: KYLIN-5792 Support ldap memberid mode
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d6285db6ab757a13ce83d119001a8ef37944522e
Author: Mingming Ge <7m...@gmail.com>
AuthorDate: Tue Aug 22 17:05:27 2023 +0800
KYLIN-5792 Support ldap memberid mode
---
.../java/org/apache/kylin/tool/util/LdapUtils.java | 12 ++++-
.../org/apache/kylin/tool/util/LdapUtilsTest.java | 57 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java b/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
index 79b621291a..ed7d3ecbca 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
@@ -112,8 +112,18 @@ public class LdapUtils {
left += maxValRange;
}
}
-
+ ldapUserDNs = rewriteUserDnIfNeeded(ldapUserDNs);
return ldapUserDNs;
}
+ public static Set<String> rewriteUserDnIfNeeded(Set<String> ldapUserDNs) {
+ String ldapUserSearchBase = KylinConfig.getInstanceFromEnv().getLDAPUserSearchBase();
+ return ldapUserDNs.stream().map(dn -> {
+ String dnName = dn;
+ if (!dn.contains(ldapUserSearchBase)) {
+ dnName = "uid=" + dn + "," + ldapUserSearchBase;
+ }
+ return dnName;
+ }).collect(Collectors.toSet());
+ }
}
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java b/src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java
new file mode 100644
index 0000000000..216d535a78
--- /dev/null
+++ b/src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.tool.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LdapUtilsTest extends NLocalFileMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ createTestMetadata();
+ }
+
+ @After
+ public void teardown() {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testRewriteUserDn() {
+ getTestConfig().setProperty("kylin.security.ldap.user-search-base", "cn=Users,dc=example,dc=com");
+ String ldapUserSearchBase = getTestConfig().getLDAPUserSearchBase();
+ Set<String> ldapUserDNs = new HashSet<>();
+ ldapUserDNs.add("a");
+ ldapUserDNs.add("b");
+ ldapUserDNs = LdapUtils.rewriteUserDnIfNeeded(ldapUserDNs);
+ Assert.assertTrue(ldapUserDNs.stream().allMatch(x -> x.contains(ldapUserSearchBase)));
+
+ Set<String> ldapUserDNs2 = new HashSet<>();
+ ldapUserDNs2.add("uid=a,cn=Users,dc=example,dc=com");
+ ldapUserDNs2.add("uid=b,cn=Users,dc=example,dc=com");
+ ldapUserDNs2 = LdapUtils.rewriteUserDnIfNeeded(ldapUserDNs2);
+ Assert.assertTrue(ldapUserDNs2.stream().allMatch(x -> x.contains(ldapUserSearchBase)));
+ }
+}
(kylin) 09/35: KYLIN-5791 & KYLIN-5801 Fix computation of timestampdff & add_month
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e33ef57cce3f3865ca9b4a6f132a170257d46be2
Author: liang.huang <83...@users.noreply.github.com>
AuthorDate: Thu Aug 31 10:46:48 2023 +0800
KYLIN-5791 & KYLIN-5801 Fix computation of timestampdff & add_month
1. Add datetype for add_month function as required
2. Support string args for Timestampdiff constant computation
---
pom.xml | 2 +-
.../scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala | 7 ++++++-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/pom.xml b/pom.xml
index ccbdfbae22..940eb93ff0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
<scala-retry>0.3.0</scala-retry>
<!-- Calcite Version -->
- <calcite.version>1.116.0-kylin-4.x-r035</calcite.version>
+ <calcite.version>1.116.0-kylin-4.x-r036</calcite.version>
<avatica.version>4.x_1.10-r01</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
index 0b7116a673..b8e34366b8 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
@@ -171,7 +171,12 @@ class SparderRexVisitor(val inputFieldNames: Seq[String],
case num: MonthNum => {
// both add_month and add_year case
val ts = k_lit(children.head).cast(TimestampType)
- return k_lit(k_add_months(k_lit(ts), num.num))
+ call.getType.getSqlTypeName match {
+ case SqlTypeName.DATE =>
+ return k_lit(k_add_months(k_lit(ts), num.num)).cast(DateType)
+ case _ =>
+ return k_lit(k_add_months(k_lit(ts), num.num))
+ }
}
case _ =>
}
(kylin) 29/35: KYLIN-5838 Replace join expression with the name of computedColumn
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit eacb62b8aa9c23a627ec5803d18b245420222f4c
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Tue Sep 26 21:23:55 2023 +0800
KYLIN-5838 Replace join expression with the name of computedColumn
---
.../kylin/query/util/ConvertToComputedColumn.java | 20 +++++++
.../apache/kylin/query/util/CCOnRealModelTest.java | 68 ++++++++++------------
2 files changed, 52 insertions(+), 36 deletions(-)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index 1c4fa5a287..8e4e7631b7 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -34,11 +34,13 @@ import javax.annotation.Nullable;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlAsOperator;
+import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlDynamicParam;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
@@ -100,12 +102,30 @@ public class ConvertToComputedColumn implements IQueryTransformer {
List<SqlNode> inputNodes = new LinkedList<>();
inputNodes.addAll(collectCandidateInputNodes(select.getSelectList(), select.getGroup()));
inputNodes.addAll(collectCandidateInputNodes(select.getOrderList(), select.getGroup()));
+ if (select.getFrom() instanceof SqlJoin) {
+ SqlJoin join = (SqlJoin) select.getFrom();
+ collectJoinNodes(inputNodes, join);
+ }
inputNodes.addAll(collectCandidateInputNode(select.getHaving(), select.getGroup()));
inputNodes.addAll(getInputTreeNodes(select.getWhere()));
inputNodes.addAll(getInputTreeNodes(select.getGroup()));
return inputNodes;
}
+ private static void collectJoinNodes(List<SqlNode> inputNodes, SqlJoin join) {
+ if (join.getLeft() instanceof SqlJoin) {
+ collectJoinNodes(inputNodes, (SqlJoin) join.getLeft());
+ }
+ SqlNode condition = join.getCondition();
+ if (condition.getKind() == SqlKind.EQUALS) {
+ SqlBasicCall call = (SqlBasicCall) condition;
+ inputNodes.addAll(call.getOperandList());
+ }
+ if (join.getRight() instanceof SqlJoin) {
+ collectJoinNodes(inputNodes, (SqlJoin) join.getRight());
+ }
+ }
+
private static List<SqlNode> collectInputNodes(SqlOrderBy sqlOrderBy) {
// if order list is not empty and query is a select
// then collect order list with checking on group keys
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
index 164810ad51..a299fbee9e 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
@@ -17,32 +17,20 @@
*/
package org.apache.kylin.query.util;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
/**
* test against real models
*/
-public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
- private ConvertToComputedColumn converter;
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata("../core-metadata/src/test/resources/ut_meta/ccjointest");
- converter = new ConvertToComputedColumn();
- }
-
- @After
- public void after() throws Exception {
- super.cleanupTestMetadata();
- }
+@MetadataInfo(overlay = "../core-metadata/src/test/resources/ut_meta/ccjointest")
+class CCOnRealModelTest {
+ private final ConvertToComputedColumn converter = new ConvertToComputedColumn();
@Test
- public void testConvertSingleTableCC() {
+ void testConvertSingleTableCC() {
{
String originSql = "select count(*), sum (price * item_count) from test_kylin_fact f left join test_order o on f.ORDER_ID = o.ORDER_ID"
+ " left join test_account a on o.buyer_id = a.account_id group by ACCOUNT_COUNTRY";
@@ -79,11 +67,10 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
check(converter, originSql, ccSql);
}
-
}
@Test
- public void testConvertCrossTableCC() {
+ void testConvertCrossTableCC() {
{
//buyer
String originSql = "select count(*), sum (price * item_count) from test_kylin_fact f left join test_order o on f.ORDER_ID = o.ORDER_ID"
@@ -112,9 +99,9 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
}
- @Ignore("historic ignored")
+ @Disabled("historic ignored")
@Test
- public void testSubquery() {
+ void testSubquery() {
{
String originSql = "select count(*), sum (price * item_count) as DEAL_AMOUNT from test_kylin_fact f left join test_order o on f.ORDER_ID = o.ORDER_ID"
+ " left join test_account a on o.buyer_id = a.account_id left join test_country c on a.account_country = c.country"
@@ -168,7 +155,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testMixModel() {
+ void testMixModel() {
String originSql = "select count(*), sum (price * item_count) from test_kylin_fact f"
+ " left join test_order o on f.ORDER_ID = o.ORDER_ID"
+ " left join test_account a on o.buyer_id = a.account_id "
@@ -197,7 +184,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testJoinOnCC() {
+ void testJoinOnCC() {
{
String originSql = "select count(*) from TEST_KYLIN_FACT\n"
+ "left join TEST_ORDER on TEST_KYLIN_FACT.ORDER_ID + 1 = TEST_ORDER.ORDER_ID + 1";
@@ -206,6 +193,16 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
check(converter, originSql, ccSql);
}
+ {
+ String originSql = "select LSTG_FORMAT_NAME, LEAF_CATEG_ID from TEST_KYLIN_FACT\n"
+ + "left join TEST_ORDER on TEST_KYLIN_FACT.ORDER_ID + 1 = TEST_ORDER.ORDER_ID + 1\n"
+ + "group by LSTG_FORMAT_NAME, LEAF_CATEG_ID";
+ String ccSql = "select LSTG_FORMAT_NAME, LEAF_CATEG_ID from TEST_KYLIN_FACT\n"
+ + "left join TEST_ORDER on TEST_KYLIN_FACT.ORDER_ID_PLUS_1 = TEST_ORDER.ID_PLUS_1\n"
+ + "group by LSTG_FORMAT_NAME, LEAF_CATEG_ID";
+ check(converter, originSql, ccSql);
+ }
+
{
String originSql = "select count(*) from TEST_KYLIN_FACT\n"
+ "left join TEST_ORDER on TEST_KYLIN_FACT.ORDER_ID + 1 = TEST_ORDER.ORDER_ID + 1\n"
@@ -230,13 +227,13 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testNoFrom() {
+ void testNoFrom() {
check(converter, "select sum(price * item_count),(SELECT 1 as VERSION) from test_kylin_fact",
"select sum(TEST_KYLIN_FACT.DEAL_AMOUNT),(SELECT 1 as VERSION) from test_kylin_fact");
}
@Test
- public void testFromValues() {
+ void testFromValues() {
String originSql = "select sum(price * item_count),(SELECT 1 FROM (VALUES(1))) from test_kylin_fact";
String ccSql = "select sum(TEST_KYLIN_FACT.DEAL_AMOUNT),(SELECT 1 FROM (VALUES(1))) from test_kylin_fact";
@@ -244,7 +241,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testNestedCC() {
+ void testNestedCC() {
String ccSql = "select count(*), sum (F.NEST4) from test_kylin_fact F";
{
@@ -269,14 +266,14 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testCcConvertedOnMultiModel() {
+ void testCcConvertedOnMultiModel() {
String originSql = "select count(*), sum (price * item_count) from test_kylin_fact f";
String ccSql = "select count(*), sum (F.DEAL_AMOUNT) from test_kylin_fact f";
check(converter, originSql, ccSql);
}
@Test
- public void testDateFamily() {
+ void testDateFamily() {
String originSql = "select count( year(date0)), max(extract(year from date1)),\n"
+ " count( month(date0)), max(extract(month from date1)),\n"
+ " count( quarter(date0)), max(extract(quarter from date1)),\n"
@@ -303,7 +300,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testBasicTimestampAddAndDiff() {
+ void testBasicTimestampAddAndDiff() {
String originSql = "select sum(timestampdiff(second, time0, time1) ) as c1,\n" //
+ "count(distinct timestampadd(minute, 1, time1)) as c2,\n" //
+ "max(timestampdiff(hour, time1, time0)) as c3,\n" //
@@ -328,7 +325,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testMoreTimestampAddAndDiff() {
+ void testMoreTimestampAddAndDiff() {
String originSql, ccSql;
originSql = "select sum((int1-int2)/(int1+int2)) as c1,\n"
+ "sum((int1-int2)/timestampdiff(second, time0, time1) ) as c2,\n"
@@ -449,7 +446,7 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testExplicitCcNameToInnerName() {
+ void testExplicitCcNameToInnerName() {
// case 1. explicit query name in inner-most sub-query
String originSql = "select max(CALCS.CC_AUTO_17)\n" + " - min(CALCS.CC_AUTO_17)\n"
@@ -493,7 +490,6 @@ public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
private void check(ConvertToComputedColumn converter, String originSql, String ccSql, String project) {
String transform = converter.transform(originSql, project, "DEFAULT");
- Assert.assertEquals(ccSql, transform);
+ Assertions.assertEquals(ccSql, transform);
}
-
}
(kylin) 11/35: KYLIN-5803 Add file segments and smart cache
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bfff6d002c9a314cd5df87ab16ff58f4a6f41fdb
Author: Zhimin Wu <59...@qq.com>
AuthorDate: Wed Apr 10 19:25:26 2024 +0800
KYLIN-5803 Add file segments and smart cache
---------
Co-authored-by: Li Yang <li...@apache.org>
Co-authored-by: Liu He <10...@qq.com>
Co-authored-by: sibing.zhang <si...@qq.com>
---
pom.xml | 28 +-
.../apache/kylin/rest/KylinPrepareEnvListener.java | 3 +-
.../apache/kylin/rest/config/AppInitializer.java | 19 +
.../config/initialize/EpochChangedListener.java | 16 +-
.../rest/request/SyncFileSegmentsRequest.java} | 23 +-
.../rest/response/SyncFileSegmentsResponse.java} | 26 +-
.../java/org/apache/kylin/common/KapConfig.java | 1 -
.../org/apache/kylin/common/KylinConfigBase.java | 35 +
.../java/org/apache/kylin/common/QueryContext.java | 18 +-
.../org/apache/kylin/common/util/ClassUtil.java | 25 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 5 +-
.../apache/kylin/common/util/ClassUtilTest.java | 63 ++
.../kylin/job/execution/NExecutableManager.java | 3 +
.../kylin/metadata/cube/model/IndexPlan.java | 2 +-
.../kylin/metadata/cube/model/NDataSegment.java | 7 +-
.../kylin/metadata/cube/model/NDataflow.java | 5 +
.../metadata/cube/model/NDataflowManager.java | 1 -
.../kylin/metadata/model/FilePartitionDesc.java} | 26 +-
.../apache/kylin/metadata/model/NDataModel.java | 5 +
.../apache/kylin/metadata/model/PartitionDesc.java | 36 +-
.../apache/kylin/metadata/model/SegmentRange.java | 86 +--
.../org/apache/kylin/metadata/model/Segments.java | 4 +
.../metadata/query/JdbcQueryHistoryStore.java | 8 +
.../metadata/query/NativeQueryRealization.java | 3 +
.../metadata/query/NoopJdbcQueryHistoryStore.java | 219 ++++++
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 6 +-
.../metadata/querymeta/SelectedColumnMeta.java | 1 -
.../rest/controller/v2/SegmentControllerV2.java | 60 +-
.../kylin/rest/service/ModelBuildService.java | 28 +-
.../apache/kylin/rest/service/ModelService.java | 43 +-
.../params/IncrementBuildSegmentParams.java | 10 +-
.../apache/kylin/query/relnode/OLAPContext.java | 19 +-
.../kylin/query/routing/QueryLayoutChooser.java | 7 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 15 +-
.../kylin/rest/controller/NQueryController.java | 100 ++-
.../rest/controller/NQueryControllerTest.java | 27 +
.../apache/kylin/rest/response/SQLResponse.java | 44 +-
.../apache/kylin/rest/response/SQLResponseV2.java | 19 +
.../apache/kylin/rest/service/QueryService.java | 11 +-
.../org/apache/kylin/query/engine/QueryExec.java | 31 +-
.../engine/exec/calcite/CalciteQueryPlanExec.java | 3 +
.../kylin/query/util/QueryContextCutter.java | 2 +-
.../org/apache/kylin/rest/QueryNodeFilter.java | 4 +
.../engine/spark/application/SparkApplication.java | 16 +-
.../kylin/engine/spark/job/MppOnTheFlyImpl.java | 160 +++++
.../engine/spark/builder/DFBuilderHelper.scala | 6 +
.../kylin/engine/spark/job/FiltersUtil.scala | 2 +-
.../kylin/engine/spark/job/FlatTableHelper.scala | 2 +-
.../kylin/engine/spark/job/SegmentBuildJob.java | 7 +-
.../apache/kylin/engine/spark/job/SegmentJob.java | 2 +-
.../engine/spark/job/stage/build/BuildStage.scala | 2 +-
.../job/stage/build/FlatTableAndDictBase.scala | 2 +-
.../stage/build/MaterializedFactTableView.scala | 32 +-
.../kylin-soft-affinity-cache-hadoop3/pom.xml | 81 +++
.../cache/kylin/KylinCacheFileSystemHadoop3.java | 43 ++
.../kylin-soft-affinity-cache/pom.xml | 26 +
.../kylin/cache/fs/AbstractCacheFileSystem.java | 277 ++++---
.../kylin/cache/fs/AlluxioHdfsFileInputStream.java | 10 +-
.../kylin/cache/fs/CacheFileSystemConstants.java | 32 +-
.../kylin/cache/fs/ManagerOfCacheFileContent.java | 116 +++
.../kylin/cache/fs/ManagerOfCacheFileStatus.java | 156 ++++
.../kylin/cache/kylin/KylinCacheFileSystem.java | 123 +++-
.../cache/kylin/OnlyForTestCacheFileSystem.java | 16 +-
.../softaffinity/SoftAffinityBookKeeping.java | 80 +++
.../cache/softaffinity/SoftAffinityConstants.java | 13 +-
.../softaffinity/SoftAffinityTextInputFormat.java | 115 +++
.../apache/kylin/cache/utils/Hadoop3CompaUtil.java | 164 +++++
.../apache/kylin/cache/utils/ReflectionUtil.java | 5 +-
.../org/apache/kylin/fileseg/FileSegments.java | 612 ++++++++++++++++
.../apache/kylin/fileseg/FileSegmentsDetector.java | 136 ++++
.../hive/serde2/lazy/LazyQuoteAwareSerDe.java | 795 +++++++++++++++++++++
.../kylin/softaffinity/SoftAffinityManager.scala | 38 +-
.../execution/datasources/CacheFilePartition.scala | 11 +-
.../datasources/CachePartitionedFile.scala | 9 +-
.../kylin/cache/fs/KylinCacheFileSystemTest.java | 135 ----
.../cache/fs/ManagerOfCacheFileContentTest.java | 74 ++
.../cache/fs/ManagerOfCacheFileStatusTest.java | 69 ++
.../cache/kylin/KylinCacheFileSystemTest.java | 203 ++++++
.../serde2/lazy/lazy/LazyQuoteAwareSerDeTest.java | 290 ++++++++
.../test/resources/default_3ywc6z78.emp/emp.csv | 3 +
.../resources/default_3ywc6z78.emp/emp_data_1.csv | 4 +
.../src/test/resources/log4j2.xml | 24 +
.../kylin/KylinCacheFileSystemWithSparkTest.scala | 140 ++++
.../HiveSoftAffinityAndLocalCacheTest.scala | 213 ++++++
src/spark-project/sparder/pom.xml | 8 +
.../query/plugin/runtime/MppOnTheFlyProvider.java | 47 ++
.../kylin/query/pushdown/SparkSqlClient.scala | 48 +-
.../kylin/query/runtime/plan/TableScanPlan.scala | 47 +-
.../scala/org/apache/spark/sql/SparderEnv.scala | 6 +-
.../sql/hive/KylinHiveSessionStateBuilder.scala | 60 +-
.../spark/sql/hive/ReplaceLocationRuleTest.scala | 12 +-
.../sql/execution/datasource/FilePruner.scala | 4 +
92 files changed, 5052 insertions(+), 521 deletions(-)
diff --git a/pom.xml b/pom.xml
index 940eb93ff0..88261b4d28 100644
--- a/pom.xml
+++ b/pom.xml
@@ -263,7 +263,17 @@
**/org/apache/kylin/tool/garbage/IndexCleaner.java,
**/org/apache/kylin/spark/common/logging/SparkExecutorHdfsLogAppender.java,
**/org/apache/kylin/rest/service/AbstractModelService.java,
- **/org/apache/spark/utils/SparkHadoopUtils.scala
+ **/org/apache/spark/utils/SparkHadoopUtils.scala,
+ **/io/kyligence/kap/fileseg/**,
+ **/org/apache/kylin/metadata/query/NoopJdbcQueryHistoryStore.java,
+ **/io/kyligence/kap/engine/spark/job/MppOnTheFlyImpl.java,
+ **/org/apache/kylin/tool/garbage/IndexCleaner.java,
+ **/io/kyligence/kap/cache/utils/Hadoop3CompaUtil.java,
+ **/io/kyligence/hive/serde2/lazy/LazyQuoteAwareSerDe.java,
+ **/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala,
+ **/org/apache/spark/sql/hive/KylinHiveSessionStateBuilder.scala,
+ **/org/apache/kylin/rest/service/ModelService.java,
+ **/org/apache/kylin/spark/common/logging/SparkExecutorHdfsLogAppender.java
</sonar.jacoco.excludes>
<sonar.coverage.exclusions>
**/org/apache/kylin/**/*Exception.java,
@@ -313,7 +323,17 @@
**/org/apache/kylin/tool/garbage/IndexCleaner.java,
**/org/apache/kylin/spark/common/logging/SparkExecutorHdfsLogAppender.java,
**/org/apache/kylin/rest/service/AbstractModelService.java,
- **/org/apache/spark/utils/SparkHadoopUtils.scala
+ **/org/apache/spark/utils/SparkHadoopUtils.scala,
+ **/io/kyligence/kap/fileseg/**,
+ **/org/apache/kylin/metadata/query/NoopJdbcQueryHistoryStore.java,
+ **/io/kyligence/kap/engine/spark/job/MppOnTheFlyImpl.java,
+ **/org/apache/kylin/tool/garbage/IndexCleaner.java,
+ **/io/kyligence/kap/cache/utils/Hadoop3CompaUtil.java,
+ **/io/kyligence/hive/serde2/lazy/LazyQuoteAwareSerDe.java,
+ **/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala,
+ **/org/apache/spark/sql/hive/KylinHiveSessionStateBuilder.scala,
+ **/org/apache/kylin/rest/service/ModelService.java,
+ **/org/apache/kylin/spark/common/logging/SparkExecutorHdfsLogAppender.java
</sonar.coverage.exclusions>
<sonar.organization>kylin</sonar.organization>
<!--suppress UnresolvedMavenProperty -->
@@ -2976,7 +2996,6 @@
<artifactId>netty</artifactId>
<version>${netty3.version}</version>
</dependency>
-
</dependencies>
</dependencyManagement>
@@ -3509,6 +3528,9 @@
<hadoop.cloud.version>3.3.1</hadoop.cloud.version>
<jetty.version>9.3.24.v20180605</jetty.version>
</properties>
+ <modules>
+ <module>src/spark-project/kylin-soft-affinity-cache-hadoop3</module>
+ </modules>
</profile>
<profile>
<id>nexus</id>
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index a49c67e33b..e4daa7fb3f 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -69,7 +69,8 @@ public class KylinPrepareEnvListener implements EnvironmentPostProcessor, Ordere
} else if (env.acceptsProfiles(Profiles.of("dev"))) {
if (env.getSystemEnvironment().containsKey(KylinConfig.KYLIN_CONF))
ClassUtil.addClasspath(env.getSystemEnvironment().get(KylinConfig.KYLIN_CONF).toString());
- if (!StringUtils.equals("true", env.getProperty("dev.diag-meta")))
+ if (env.getProperty("dev.dont-load-test-metadata") == null
+ && !StringUtils.equals("true", env.getProperty("dev.diag-meta")))
setLocalEnvs();
}
// enable CC check
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
index 35af91a415..c58b5169fc 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
@@ -65,11 +65,15 @@ import org.apache.kylin.tool.garbage.PriorityExecutor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.TaskScheduler;
+import org.springframework.security.web.FilterChainProxy;
+import org.springframework.security.web.firewall.DefaultHttpFirewall;
+import org.springframework.security.web.firewall.HttpFirewall;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -171,6 +175,15 @@ public class AppInitializer {
EventBusFactory.getInstance().register(new QueryMetricsListener(), false);
EventBusFactory.getInstance().register(new UserAclListener(), true);
+ if (kylinConfig.isAllowNonAsciiCharInUrl()) {
+ // Note: DefaultHttpFirewall vs StrictHttpFirewall
+ // In order to allow Chinese chars on URL like "/{cubeName}/segments", we have to use DefaultHttpFirewall.
+ // If later we have to use StrictHttpFirewall, then StrictHttpFirewall.rejectNonPrintableAsciiCharactersInFieldName()
+ // must be overridden to allow Chinese chars on URL.
+ FilterChainProxy filterChainProxy = context.getBean(FilterChainProxy.class);
+ filterChainProxy.setFirewall(this.getHttpFirewall());
+ }
+
postInit();
log.info("Kylin initialization completed.");
@@ -240,4 +253,10 @@ public class AppInitializer {
log.warn("set Fs URL stream handler factory failed", e);
}
}
+
+ @Bean
+ public HttpFirewall getHttpFirewall() {
+ return new DefaultHttpFirewall();
+ }
+
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/EpochChangedListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/EpochChangedListener.java
index 8b7246806a..7b752853d0 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/EpochChangedListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/EpochChangedListener.java
@@ -162,19 +162,23 @@ public class EpochChangedListener {
if (!scheduler.hasStarted()) {
throw new KylinRuntimeException("Scheduler for " + project + " has not been started");
}
+
StreamingScheduler ss = StreamingScheduler.getInstance(project);
ss.init();
if (!ss.getHasStarted().get()) {
throw new KylinRuntimeException("Streaming Scheduler for " + project + " has not been started");
}
- QueryHistoryTaskScheduler qhAccelerateScheduler = QueryHistoryTaskScheduler.getInstance(project);
- qhAccelerateScheduler.init();
- if (!qhAccelerateScheduler.hasStarted()) {
- throw new KylinRuntimeException(
- "Query history accelerate scheduler for " + project + " has not been started");
+ if (kylinConfig.getQueryHistoryAccelerateInterval() > 0) {
+ QueryHistoryTaskScheduler qhAccelerateScheduler = QueryHistoryTaskScheduler.getInstance(project);
+ qhAccelerateScheduler.init();
+
+ if (!qhAccelerateScheduler.hasStarted()) {
+ throw new KylinRuntimeException(
+ "Query history accelerate scheduler for " + project + " has not been started");
+ }
+ recommendationUpdateScheduler.addProject(project);
}
- recommendationUpdateScheduler.addProject(project);
return 0;
}, project, 1);
scheduler.setHasFinishedTransactions(new AtomicBoolean(true));
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java b/src/common-service/src/main/java/org/apache/kylin/rest/request/SyncFileSegmentsRequest.java
similarity index 68%
copy from src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
copy to src/common-service/src/main/java/org/apache/kylin/rest/request/SyncFileSegmentsRequest.java
index a18e5e6ee9..d7a175a63c 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/request/SyncFileSegmentsRequest.java
@@ -15,19 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.cache.kylin;
+package org.apache.kylin.rest.request;
-import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
+import com.fasterxml.jackson.annotation.JsonProperty;
-public class OnlyForTestCacheFileSystem extends AbstractCacheFileSystem {
+import java.util.List;
+import lombok.Data;
-
- /**
- * Check whether needs to cache data on the current executor
- */
- @Override
- public boolean isUseLocalCacheForTargetExecs() {
- return true;
- }
+@Data
+public class SyncFileSegmentsRequest {
+ @JsonProperty("project")
+ private String project;
+ @JsonProperty("sql")
+ private String sql;
+ @JsonProperty("models_or_tables")
+ private List<String> modelAliasOrFactTables;
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/SyncFileSegmentsResponse.java
similarity index 62%
copy from src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
copy to src/common-service/src/main/java/org/apache/kylin/rest/response/SyncFileSegmentsResponse.java
index a18e5e6ee9..db48d8b544 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/SyncFileSegmentsResponse.java
@@ -15,19 +15,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.cache.kylin;
+package org.apache.kylin.rest.response;
-import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
+import org.apache.kylin.fileseg.FileSegments.ModelFileSegments;
-public class OnlyForTestCacheFileSystem extends AbstractCacheFileSystem {
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class SyncFileSegmentsResponse {
+ @JsonProperty("project")
+ private String project;
+ @JsonProperty("models")
+ private List<ModelFileSegments> models;
- /**
- * Check whether needs to cache data on the current executor
- */
- @Override
- public boolean isUseLocalCacheForTargetExecs() {
- return true;
- }
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index a526310691..c7c10c500d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -730,5 +730,4 @@ public class KapConfig {
public long getBigQueryThresholdUpdateIntervalSecond() {
return Long.parseLong(config.getOptional("kylin.query.big-query-threshold-update-interval-second", "10800"));
}
-
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 64b4a775d4..f51bb8ba8a 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -346,6 +346,10 @@ public abstract class KylinConfigBase implements Serializable {
return "UT".equals(getDeployEnv());
}
+ /**
+ * @deprecated Use <code>isUTEnv()</code> or <code>isDevOrUT()</code> instead
+ */
+ @Deprecated // use isUTEnv() or isDevOrUT() instead
public boolean isDevEnv() {
return "DEV".equals(getDeployEnv());
}
@@ -1907,6 +1911,14 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.schema-cache-enabled", FALSE));
}
+ public boolean isMppOnTheFlyLayoutsEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.mpp-on-the-fly-layouts-enabled", FALSE));
+ }
+
+ public String getMppOnTheFlyLayoutsProvider() {
+ return this.getOptional("kylin.query.mpp-on-the-fly-layouts-provider", "io.kyligence.kap.engine.spark.job.MppOnTheFlyImpl");
+ }
+
public boolean enableReplaceDynamicParams() {
return Boolean.parseBoolean(this.getOptional("kylin.query.replace-dynamic-params-enabled", FALSE));
}
@@ -2167,6 +2179,10 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.auto-set-shuffle-partitions-enabled", TRUE));
}
+ public int autoSetPushDownPartitionsForced() {
+ return Integer.parseInt(this.getOptional("kylin.query.pushdown.auto-set-shuffle-partitions-forced", "0"));
+ }
+
public int getBaseShufflePartitionSize() {
return Integer.parseInt(this.getOptional("kylin.query.pushdown.base-shuffle-partition-size", "48"));
}
@@ -2374,6 +2390,10 @@ public abstract class KylinConfigBase implements Serializable {
return timeZone;
}
+ public boolean isAllowNonAsciiCharInUrl() {
+ return Boolean.parseBoolean(this.getOptional("kylin.web.allow-non-ascii-char-in-url", FALSE));
+ }
+
// ============================================================================
// RESTCLIENT
// ============================================================================
@@ -4068,4 +4088,19 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.query.round-decimal-zero", FALSE));
}
+ public boolean isKylinLocalCacheEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.columnar.spark-conf.spark.kylin.local-cache.enabled", FALSE));
+ }
+
+ public boolean isKylinFileStatusCacheEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.storage.columnar.spark-conf.spark.hadoop.spark.kylin.file-status-cache.enabled", FALSE));
+ }
+
+ public boolean isSupportPushdownHiveCsvEnhancement() {
+ return Boolean.parseBoolean(getOptional("kylin.query.pushdown.hive.csv.enhancement", FALSE));
+ }
+
+ public String getFileSegmentSuccessFile() {
+ return getOptional("kylin.fileseg.success-file", null);
+ }
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index d364fd2526..592df87f68 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -29,6 +29,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
@@ -42,6 +43,7 @@ import lombok.Setter;
/**
* Holds per query information and statistics.
*/
+@Slf4j
public class QueryContext implements Closeable {
public static final String PUSHDOWN_RDBMS = "RDBMS";
@@ -121,7 +123,7 @@ public class QueryContext implements Closeable {
// record second storage partition which used
@Getter
- private List<Integer> usedPartitionIndexes = new ArrayList<>();
+ private final List<Integer> usedPartitionIndexes = new ArrayList<>();
// record second storage partition which used
@Getter
@@ -202,7 +204,7 @@ public class QueryContext implements Closeable {
public void record(String message) {
long current = System.currentTimeMillis();
long takeTime = current - recordMillis;
- queryRecord.put(message, takeTime + "");
+ queryRecord.put(message, Long.toString(takeTime));
recordMillis = current;
}
@@ -242,7 +244,7 @@ public class QueryContext implements Closeable {
// ============================================================================
/**
- * query metrics
+ * query metrics, spark execution related
*/
@Getter
@Setter
@@ -272,6 +274,7 @@ public class QueryContext implements Closeable {
private long queryStageCount;
private long queryTaskCount;
private int retryTimes;
+ private long dataFetchTime; // see doc in SQLResponse.dataFetchTime
private String queryExecutedPlan;
// sourceScanXxx from Parquet
@@ -309,6 +312,10 @@ public class QueryContext implements Closeable {
retryTimes += 1;
}
+ public void addDataFetchTime(long dataFetchTime) {
+ this.dataFetchTime = Long.max(this.dataFetchTime, dataFetchTime);
+ }
+
// scanXxx from SparkUI
@Getter
@Setter
@@ -334,7 +341,6 @@ public class QueryContext implements Closeable {
}
/**
- * @param scanList
* @return if scanList == null return default -1, else return sum of list
*/
public static long calValueWithDefault(List<Long> scanList) {
@@ -397,8 +403,8 @@ public class QueryContext implements Closeable {
private Long layoutId;
private String indexType;
private boolean isPartialMatchModel;
- private boolean isValid = true;
- private boolean isLayoutExist = true;
+ private boolean isValid;
+ private boolean isLayoutExist;
private boolean isStreamingLayout;
private List<String> snapshots;
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index c4d31c3aba..30de8836a5 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -26,6 +26,7 @@ import java.net.URLClassLoader;
import java.net.URLDecoder;
import java.util.Enumeration;
+import org.apache.commons.lang3.Validate;
import org.slf4j.LoggerFactory;
/**
@@ -40,14 +41,34 @@ public class ClassUtil {
public static void addToClasspath(String path, ClassLoader classLoader) {
logger.info("Adding path " + path + " to class path");
- File file = new File(path);
+ String[] paths = path.split("[,:]");
+ for (String p : paths) {
+ File file = new File(p);
+ if (file.exists()) {
+ _addToClasspath(file, classLoader);
+ } else {
+ File dir = file.getParentFile();
+ Validate.isTrue(dir.exists());
+ String match = file.getName();
+ Validate.isTrue(match.startsWith("*") && match.endsWith("*"));
+ match = match.substring(1, match.length() - 1);
+ for (File f : dir.listFiles()) {
+ if (f.getName().contains(match) && f.getName().endsWith(".jar"))
+ _addToClasspath(f, classLoader);
+ }
+ }
+ }
+ }
+
+ private static void _addToClasspath(File file, ClassLoader classLoader) {
try {
if (file.exists()) {
+ logger.info("Adding path {} to class path", file);
Class<URLClassLoader> urlClass = URLClassLoader.class;
Method method = urlClass.getDeclaredMethod("addURL", URL.class);
Unsafe.changeAccessibleObject(method, true);
- method.invoke((URLClassLoader) classLoader, file.toURI().toURL());
+ method.invoke(classLoader, file.toURI().toURL());
}
} catch (Exception e) {
throw new RuntimeException(e);
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 0e3ac41ae5..6e8f358a24 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -111,9 +111,12 @@ public class HadoopUtil {
* @return
*/
public static Configuration getHadoopConfFromSparkEngine() {
- val conf = new Configuration(false);
val hadoopConfDir = getHadoopConfDir();
+ if (StringUtils.isBlank(hadoopConfDir)) {
+ return new Configuration();
+ }
+ val conf = new Configuration(false);
conf.addResource(new Path(Paths.get(hadoopConfDir, "core-site.xml").toFile().getAbsolutePath()));
conf.addResource(new Path(Paths.get(hadoopConfDir, "hdfs-site.xml").toFile().getAbsolutePath()));
conf.addResource(new Path(Paths.get(hadoopConfDir, "yarn-site.xml").toFile().getAbsolutePath()));
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java b/src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
index 0a0906e2cc..6f456b2197 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
@@ -18,8 +18,18 @@
package org.apache.kylin.common.util;
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.io.FileUtils;
import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
public class ClassUtilTest {
@@ -31,4 +41,57 @@ public class ClassUtilTest {
.contains("jcl-over-slf4j"));
}
+ @Test
+ void testAddToClasspath() throws Exception {
+ // Prepare tmp dir & jars
+ File extraJarsDir = null;
+ try {
+ String tmpDir = System.getProperty("java.io.tmpdir");
+ extraJarsDir = new File(tmpDir, "class_util_test_jars");
+ extraJarsDir.mkdir();
+
+ List<File> jarFiles = Arrays.asList(new File(extraJarsDir, "hive-common-3.3.1.jar"),
+ new File(extraJarsDir, "aws-java-sdk-core-1.11.901.jar"),
+ new File(extraJarsDir, "hadoop-aws-3.3.1.jar"),
+ new File(extraJarsDir, "hadoop-aws-3.3.1"),
+ new File(extraJarsDir, "no-match-file.jar"));
+ for (File f : jarFiles) {
+ f.createNewFile();
+ }
+ File singleJarFile = new File(extraJarsDir, "a-single-file.jar");
+ singleJarFile.createNewFile();
+
+ // Load jars
+ ClassLoader loader = Thread.currentThread().getContextClassLoader();
+ ClassUtil.addToClasspath(String.format("%s/*hive*", extraJarsDir.getCanonicalPath()), loader);
+ ClassUtil.addToClasspath(String.format("%s/*aws*", extraJarsDir.getCanonicalPath()), loader);
+ try {
+ ClassUtil.addToClasspath(String.format("%s/*hive", extraJarsDir.getCanonicalPath()), loader);
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ }
+ try {
+ ClassUtil.addToClasspath(String.format("%s/hive*", extraJarsDir.getCanonicalPath()), loader);
+ } catch (Exception e) {
+ Assertions.assertTrue(e instanceof IllegalArgumentException);
+ }
+ ClassUtil.addToClasspath(singleJarFile.getCanonicalPath(), loader);
+
+ URL[] urls = (URL[]) ReflectionTestUtils.invokeMethod(loader, "getURLs");
+ List<URL> urlList = Stream.of(urls).filter(url -> {
+ for (File f : jarFiles) {
+ if (url.getPath().contains(f.getName()) || url.getPath().contains(singleJarFile.getName())) {
+ return true;
+ }
+ }
+ return false;
+ }).collect(Collectors.toList());
+ Assertions.assertEquals(4, urlList.size());
+ } finally {
+ if (extraJarsDir != null) {
+ FileUtils.forceDeleteOnExit(extraJarsDir);
+ }
+ }
+ }
+
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
index bcac101b89..5eca8383e9 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
@@ -1198,6 +1198,9 @@ public class NExecutableManager {
ExecutablePO stage = stageMapFromSegment.getOrDefault(segmentId, Lists.newArrayList()).stream()
.filter(po -> po.getId().equals(taskOrJobId))//
.findFirst().orElse(null);
+ if (stage == null) {
+ return false; // local spark job?
+ }
ExecutableOutputPO stageOutput = stage.getOutput();
assertOutputNotNull(stageOutput, taskOrJobId, segmentId);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
index e44fa1b838..ee80805d1a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
@@ -1266,7 +1266,7 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
if (layoutInIndexPlan.isAuto()) {
indexEntity.getLayouts().remove(layoutInIndexPlan);
whiteIndexesMap.values().stream().filter(
- indexEntityInIndexPlan -> indexEntityInIndexPlan.getId() == indexEntity.getId())
+ indexEntityInIndexPlan -> indexEntityInIndexPlan.getId() == indexEntity.getId())
.findFirst().ifPresent(indexEntityInIndexPlan -> indexEntityInIndexPlan.getLayouts()
.remove(layoutInIndexPlan));
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
index aa9109efa4..2337b6e767 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegment.java
@@ -53,7 +53,7 @@ import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.Getter;
import lombok.Setter;
-@SuppressWarnings("serial")
+@SuppressWarnings({"rawtypes", "FieldMayBeFinal"})
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class NDataSegment implements ISegment, Serializable {
@@ -160,6 +160,7 @@ public class NDataSegment implements ISegment, Serializable {
layoutInfo = new LayoutInfo();
}
+ @SuppressWarnings("CopyConstructorMissesField")
public NDataSegment(NDataSegment other) {
this.id = other.id;
this.name = other.name;
@@ -178,7 +179,7 @@ public class NDataSegment implements ISegment, Serializable {
layoutInfo = new LayoutInfo(other.getSegDetails());
}
- public <T extends Comparable> NDataSegment(NDataflow df, SegmentRange<T> segmentRange) {
+ public <T extends Comparable<?>> NDataSegment(NDataflow df, SegmentRange<T> segmentRange) {
this.dataflow = df;
this.segmentRange = segmentRange;
this.name = Segments.makeSegmentName(segmentRange);
@@ -188,7 +189,7 @@ public class NDataSegment implements ISegment, Serializable {
this.layoutInfo = new LayoutInfo();
}
- public <T extends Comparable> NDataSegment(NDataflow df, SegmentRange<T> segRange, String id) {
+ public <T extends Comparable<?>> NDataSegment(NDataflow df, SegmentRange<T> segRange, String id) {
this(df, segRange);
if (!StringUtils.isEmpty(id)) {
this.id = id;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java
index 9c7542492a..595abcd619 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflow.java
@@ -107,6 +107,11 @@ public class NDataflow extends RootPersistentEntity implements Serializable, IRe
@JsonProperty("last_query_time")
private long lastQueryTime = 0L;
+ @Getter
+ @Setter
+ @JsonProperty("last_data_refresh_time")
+ private long lastDataRefreshTime = 0L;
+
@Getter
@Setter
@JsonProperty("layout_query_hit_count")
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
index f38ed45ed6..022d7a39c4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
@@ -395,7 +395,6 @@ public class NDataflowManager implements IRealizationProvider {
}
public NDataSegment refreshSegment(NDataflow df, SegmentRange segRange) {
-
NDataSegment newSegment = newSegment(df, segRange);
NDataSegment toRefreshSeg = null;
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FilePartitionDesc.java
similarity index 52%
copy from src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
copy to src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FilePartitionDesc.java
index 0a0906e2cc..c8e77dfa6c 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/util/ClassUtilTest.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FilePartitionDesc.java
@@ -16,19 +16,21 @@
* limitations under the License.
*/
-package org.apache.kylin.common.util;
+package org.apache.kylin.metadata.model;
-import org.junit.Assert;
-import org.junit.jupiter.api.Test;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
-public class ClassUtilTest {
-
- @Test
- public void testFindContainingJar() throws ClassNotFoundException {
- Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.logging.LogFactory"))
- .contains("commons-logging"));
- Assert.assertTrue(ClassUtil.findContainingJar(Class.forName("org.apache.commons.logging.LogFactory"), "slf4j")
- .contains("jcl-over-slf4j"));
- }
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
+public class FilePartitionDesc extends PartitionDesc {
+ @Getter
+ @Setter
+ @EqualsAndHashCode.Include
+ @JsonProperty("file_storage_location")
+ private String fileStorageLocation; // used by FILE based partition mode
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 9366cbf387..99c0d6c908 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -62,6 +62,7 @@ import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.graph.JoinsGraph;
+import org.apache.kylin.metadata.model.PartitionDesc.PartitionType;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -1527,4 +1528,8 @@ public class NDataModel extends RootPersistentEntity {
return getEffectiveMeasures().values().stream().filter(m -> m.getType() == NDataModel.MeasureType.INTERNAL)
.map(NDataModel.Measure::getId).collect(Collectors.toSet());
}
+
+ public boolean isFilePartitioned() {
+ return partitionDesc != null && partitionDesc.getCubePartitionType() == PartitionType.FILE;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index e176c3047b..88789fb750 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -31,25 +31,25 @@ import org.apache.kylin.common.exception.JobErrorCode;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.EqualsAndHashCode;
/**
*/
-@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class PartitionDesc implements Serializable {
- private static String and = " AND ";
+ private static final String SPACE_AND_SPACE = " AND ";
- public static enum PartitionType implements Serializable {
+ public enum PartitionType implements Serializable {
+ FILE, // marks FILE based partition mode
APPEND, //
UPDATE_INSERT // not used since 0.7.1
}
@@ -78,12 +78,14 @@ public class PartitionDesc implements Serializable {
private IPartitionConditionBuilder partitionConditionBuilder;
public void init(NDataModel model) {
- if (StringUtils.isEmpty(partitionDateColumn))
- return;
+ if (!StringUtils.isEmpty(partitionDateColumn)) {
+ partitionDateColumnRef = model.findColumn(partitionDateColumn);
+ partitionDateColumn = partitionDateColumnRef.getIdentity();
+ }
- partitionDateColumnRef = model.findColumn(partitionDateColumn);
- partitionDateColumn = partitionDateColumnRef.getIdentity();
- partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
+ if (!StringUtils.isEmpty(partitionConditionBuilderClz)) {
+ partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
+ }
}
public boolean partitionColumnIsYmdInt() {
@@ -229,6 +231,7 @@ public class PartitionDesc implements Serializable {
// ============================================================================
+ @SuppressWarnings("rawtypes")
public interface IPartitionConditionBuilder {
String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange);
@@ -236,6 +239,7 @@ public class PartitionDesc implements Serializable {
final LinkedList<Long> partitionIds, final ISegment seg, final SegmentRange segRange);
}
+ @SuppressWarnings("rawtypes")
public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder, Serializable {
public DefaultPartitionConditionBuilder() {
@@ -310,7 +314,7 @@ public class PartitionDesc implements Serializable {
String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ").append(String.format(Locale.ROOT, "to_date('%s', '%s')",
DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat), partitionColumnDateFormat));
- builder.append(and);
+ builder.append(SPACE_AND_SPACE);
builder.append(partitionColumnName).append(" < ").append(String.format(Locale.ROOT, "to_date('%s', '%s')",
DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat), partitionColumnDateFormat));
}
@@ -319,7 +323,7 @@ public class PartitionDesc implements Serializable {
long startInclusive, long endExclusive) {
String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ").append(startInclusive);
- builder.append(and);
+ builder.append(SPACE_AND_SPACE);
builder.append(partitionColumnName).append(" < ").append(endExclusive);
}
@@ -328,7 +332,7 @@ public class PartitionDesc implements Serializable {
String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ")
.append(DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_MONTH_PATTERN));
- builder.append(and);
+ builder.append(SPACE_AND_SPACE);
builder.append(partitionColumnName).append(" < ")
.append(DateFormat.formatToDateStr(endExclusive, DateFormat.COMPACT_MONTH_PATTERN));
}
@@ -338,7 +342,7 @@ public class PartitionDesc implements Serializable {
String partitionColumnName = partitionColumn.getBackTickExp();
builder.append(partitionColumnName).append(" >= ")
.append(DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN));
- builder.append(and);
+ builder.append(SPACE_AND_SPACE);
builder.append(partitionColumnName).append(" < ")
.append(DateFormat.formatToDateStr(endExclusive, DateFormat.COMPACT_DATE_PATTERN));
}
@@ -352,8 +356,8 @@ public class PartitionDesc implements Serializable {
return;
}
- String startInc = null;
- String endInc = null;
+ String startInc;
+ String endInc;
if (StringUtils.isBlank(partitionColumnDateFormat)) {
startInc = String.valueOf(startInclusive);
endInc = String.valueOf(endExclusive);
@@ -363,7 +367,7 @@ public class PartitionDesc implements Serializable {
}
builder.append(partitionColumnName).append(" >= '").append(startInc).append("'");
- builder.append(" AND ");
+ builder.append(SPACE_AND_SPACE);
builder.append(partitionColumnName).append(" < '").append(endInc).append("'");
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
index ad54b1e8c4..f48cfff0bb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentRange.java
@@ -45,45 +45,45 @@ import lombok.val;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class")
-abstract public class SegmentRange<T extends Comparable> implements Comparable<SegmentRange>, Serializable {
+public abstract class SegmentRange<T extends Comparable<?>> implements Comparable<SegmentRange<T>>, Serializable {
protected T start;
protected T end;
- abstract public boolean isInfinite();
+ public abstract boolean isInfinite();
- abstract public boolean contains(SegmentRange o);
+ public abstract boolean contains(SegmentRange<T> o);
- abstract public boolean entireOverlaps(SegmentRange o);
+ public abstract boolean entireOverlaps(SegmentRange<T> o);
- abstract public boolean overlaps(SegmentRange o);
+ public abstract boolean overlaps(SegmentRange<T> o);
- abstract public boolean connects(SegmentRange o);
+ public abstract boolean connects(SegmentRange<T> o);
- abstract public boolean apartBefore(SegmentRange o);
+ public abstract boolean apartBefore(SegmentRange<T> o);
- abstract public boolean startStartMatch(SegmentRange o);
+ public abstract boolean startStartMatch(SegmentRange<T> o);
- abstract public boolean endEndMatch(SegmentRange o);
+ public abstract boolean endEndMatch(SegmentRange<T> o);
- abstract public boolean startEndMatch(SegmentRange o);
+ public abstract boolean startEndMatch(SegmentRange<T> o);
- abstract public SegmentRange getStartDeviation(SegmentRange o);
+ public abstract SegmentRange<T> getStartDeviation(SegmentRange<T> o);
- abstract public SegmentRange getEndDeviation(SegmentRange o);
+ public abstract SegmentRange<T> getEndDeviation(SegmentRange<T> o);
- abstract public SegmentRange getOverlapRange(SegmentRange o);
+ public abstract SegmentRange<T> getOverlapRange(SegmentRange<T> o);
/**
* create a new SegmentRange which will start from this.start and end at o.end
* caller should make sure this.start < o.end
*/
- abstract public SegmentRange coverWith(SegmentRange o);
+ public abstract SegmentRange<T> coverWith(SegmentRange<T> o);
/**
* create a new SegmentRange which will start from this.end and end at o.start
* caller should make sure this.end < o.start
*/
- abstract public SegmentRange gapTill(SegmentRange o);
+ public abstract SegmentRange<T> gapTill(SegmentRange<T> o);
public T getStart() {
return start;
@@ -101,9 +101,13 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
return end.toString();
}
+ public String proposeSegmentName() {
+ return null;
+ }
+
// ============================================================================
- abstract public static class BasicSegmentRange extends SegmentRange<Long> {
+ public abstract static class BasicSegmentRange extends SegmentRange<Long> {
BasicSegmentRange() {
}
@@ -114,12 +118,12 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
Preconditions.checkState(this.start <= this.end);
}
- private BasicSegmentRange convert(SegmentRange o) {
+ private BasicSegmentRange convert(SegmentRange<Long> o) {
Preconditions.checkState(o instanceof BasicSegmentRange);
return (BasicSegmentRange) o;
}
- private void checkSameType(SegmentRange o) {
+ protected void checkSameType(SegmentRange<Long> o) {
Preconditions.checkNotNull(o);
Preconditions.checkState(getClass() == o.getClass());
}
@@ -130,55 +134,55 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
}
@Override
- public boolean contains(SegmentRange o) {
+ public boolean contains(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.start <= t.start && t.end <= this.end;
}
@Override
- public boolean entireOverlaps(SegmentRange o) {
+ public boolean entireOverlaps(SegmentRange<Long> o) {
checkSameType(o);
return this.contains(o) && o.contains(this);
}
@Override
- public boolean overlaps(SegmentRange o) {
+ public boolean overlaps(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.start < t.end && t.start < this.end;
}
@Override
- public boolean connects(SegmentRange o) {
+ public boolean connects(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.end.equals(t.start);
}
@Override
- public boolean apartBefore(SegmentRange o) {
+ public boolean apartBefore(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.end < t.start;
}
@Override
- public boolean startStartMatch(SegmentRange o) {
+ public boolean startStartMatch(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.start.equals(t.start);
}
@Override
- public boolean endEndMatch(SegmentRange o) {
+ public boolean endEndMatch(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.end.equals(t.end);
}
@Override
- public boolean startEndMatch(SegmentRange o) {
+ public boolean startEndMatch(SegmentRange<Long> o) {
checkSameType(o);
BasicSegmentRange t = convert(o);
return this.start.equals(t.end);
@@ -205,7 +209,7 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
}
@Override
- public int compareTo(SegmentRange o) {
+ public int compareTo(SegmentRange<Long> o) {
BasicSegmentRange t = convert(o);
int comp = Long.compare(this.start, t.start);
@@ -234,31 +238,31 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
return new TimePartitionedSegmentRange(0L, Long.MAX_VALUE);
}
- private TimePartitionedSegmentRange convertToTimePartitioned(SegmentRange o) {
+ private TimePartitionedSegmentRange convertToTimePartitioned(SegmentRange<Long> o) {
Preconditions.checkState(o instanceof TimePartitionedSegmentRange);
return (TimePartitionedSegmentRange) o;
}
@Override
- public SegmentRange coverWith(SegmentRange o) {
+ public SegmentRange<Long> coverWith(SegmentRange<Long> o) {
TimePartitionedSegmentRange other = convertToTimePartitioned(o);
return new TimePartitionedSegmentRange(this.start, other.end);
}
@Override
- public SegmentRange getStartDeviation(SegmentRange o) {
+ public SegmentRange<Long> getStartDeviation(SegmentRange<Long> o) {
TimePartitionedSegmentRange other = convertToTimePartitioned(o);
return new TimePartitionedSegmentRange(this.start, other.start);
}
@Override
- public SegmentRange getEndDeviation(SegmentRange o) {
+ public SegmentRange<Long> getEndDeviation(SegmentRange<Long> o) {
TimePartitionedSegmentRange other = convertToTimePartitioned(o);
return new TimePartitionedSegmentRange(this.end, other.end);
}
@Override
- public SegmentRange getOverlapRange(SegmentRange o) {
+ public SegmentRange<Long> getOverlapRange(SegmentRange<Long> o) {
TimePartitionedSegmentRange other = convertToTimePartitioned(o);
if (!this.overlaps(o)) {
return null;
@@ -269,7 +273,7 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
}
@Override
- public SegmentRange gapTill(SegmentRange o) {
+ public SegmentRange<Long> gapTill(SegmentRange<Long> o) {
TimePartitionedSegmentRange other = convertToTimePartitioned(o);
return new TimePartitionedSegmentRange(this.end, other.start);
}
@@ -333,20 +337,20 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
: sourcePartitionOffsetEnd;
}
- private KafkaOffsetPartitionedSegmentRange convertToKafkaOffset(SegmentRange o) {
+ private KafkaOffsetPartitionedSegmentRange convertToKafkaOffset(SegmentRange<Long> o) {
Preconditions.checkState(o instanceof KafkaOffsetPartitionedSegmentRange);
return (KafkaOffsetPartitionedSegmentRange) o;
}
@Override
- public boolean contains(SegmentRange o) {
+ public boolean contains(SegmentRange<Long> o) {
KafkaOffsetPartitionedSegmentRange other = convertToKafkaOffset(o);
return comparePartitionOffset(this.sourcePartitionOffsetStart, other.sourcePartitionOffsetStart) <= 0
&& comparePartitionOffset(other.sourcePartitionOffsetEnd, this.sourcePartitionOffsetEnd) <= 0;
}
@Override
- public SegmentRange coverWith(SegmentRange o) {
+ public SegmentRange<Long> coverWith(SegmentRange<Long> o) {
KafkaOffsetPartitionedSegmentRange other = convertToKafkaOffset(o);
val start = Math.min(this.start, other.start);
val end = Math.max(this.end, other.end);
@@ -355,28 +359,28 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
}
@Override
- public SegmentRange gapTill(SegmentRange o) {
+ public SegmentRange<Long> gapTill(SegmentRange<Long> o) {
KafkaOffsetPartitionedSegmentRange other = convertToKafkaOffset(o);
return new KafkaOffsetPartitionedSegmentRange(this.end, other.start, this.getSourcePartitionOffsetEnd(),
other.getSourcePartitionOffsetStart());
}
@Override
- public SegmentRange getStartDeviation(SegmentRange o) {
+ public SegmentRange<Long> getStartDeviation(SegmentRange<Long> o) {
KafkaOffsetPartitionedSegmentRange other = convertToKafkaOffset(o);
return new KafkaOffsetPartitionedSegmentRange(this.start, other.start, this.getSourcePartitionOffsetStart(),
other.getSourcePartitionOffsetStart());
}
@Override
- public SegmentRange getEndDeviation(SegmentRange o) {
+ public SegmentRange<Long> getEndDeviation(SegmentRange<Long> o) {
KafkaOffsetPartitionedSegmentRange other = convertToKafkaOffset(o);
return new KafkaOffsetPartitionedSegmentRange(this.end, other.end, this.getSourcePartitionOffsetEnd(),
other.getSourcePartitionOffsetEnd());
}
@Override
- public boolean overlaps(SegmentRange o) {
+ public boolean overlaps(SegmentRange<Long> o) {
super.checkSameType(o);
KafkaOffsetPartitionedSegmentRange t = convertToKafkaOffset(o);
@@ -389,7 +393,7 @@ abstract public class SegmentRange<T extends Comparable> implements Comparable<S
}
@Override
- public SegmentRange getOverlapRange(SegmentRange o) {
+ public SegmentRange<Long> getOverlapRange(SegmentRange<Long> o) {
KafkaOffsetPartitionedSegmentRange other = convertToKafkaOffset(o);
if (!this.overlaps(o)) {
return null;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 13e1b1bac2..c88853b39a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -727,6 +727,10 @@ public class Segments<T extends ISegment> extends ArrayList<T> implements Serial
return "FULL_BUILD";
}
+ String proposedName = segRange.proposeSegmentName();
+ if (proposedName != null)
+ return proposedName;
+
if (segRange instanceof SegmentRange.TimePartitionedSegmentRange) {
// using time
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss",
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index e35aae28f5..4349dbd8a1 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -116,6 +116,14 @@ public class JdbcQueryHistoryStore {
sqlSessionFactory = QueryHisStoreUtil.getSqlSessionFactory(dataSource, qhTableName, qhRealizationTableName);
}
+ // for subclass NoopJdbcQueryHistoryStore
+ JdbcQueryHistoryStore() {
+ queryHistoryTable = null;
+ queryHistoryRealizationTable = null;
+ sqlSessionFactory = null;
+ dataSource = null;
+ }
+
public void dropQueryHistoryTable() throws SQLException {
try (Connection connection = dataSource.getConnection()) {
ScriptRunner sr = new ScriptRunner(connection);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java
index 588724faf8..48c54eedad 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NativeQueryRealization.java
@@ -46,6 +46,9 @@ public class NativeQueryRealization implements Serializable {
private boolean recommendSecondStorage = false;
private boolean isStreamingLayout = false;
private List<String> snapshots;
+ private long lastDataRefreshTime;
+ private boolean isLoadingData;
+ private boolean isBuildingIndex;
@JsonUnwrapped
@Getter
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NoopJdbcQueryHistoryStore.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NoopJdbcQueryHistoryStore.java
new file mode 100644
index 0000000000..6cbfced03e
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/NoopJdbcQueryHistoryStore.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.query;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.query.QueryMetrics.RealizationMetrics;
+import org.mybatis.dynamic.sql.insert.render.InsertStatementProvider;
+import org.mybatis.dynamic.sql.select.QueryExpressionDSL;
+import org.mybatis.dynamic.sql.select.SelectModel;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class NoopJdbcQueryHistoryStore extends JdbcQueryHistoryStore {
+
+ public NoopJdbcQueryHistoryStore() {
+ // nothing to do
+ }
+
+ @Override
+ public void dropQueryHistoryTable() {
+ // nothing to do
+ }
+
+ @Override
+ public int insert(QueryMetrics queryMetrics) {
+ return 0;
+ }
+
+ @Override
+ public void insert(List<QueryMetrics> queryMetricsList) {
+ // nothing to do
+ }
+
+ @Override
+ public List<QueryHistory> queryQueryHistoriesByConditions(QueryHistoryRequest request, int limit, int offset) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public QueryStatistics queryQueryHistoriesSize(QueryHistoryRequest request) {
+ return new QueryStatistics();
+ }
+
+ @Override
+ public List<QueryDailyStatistic> queryHistoryDailyStatistic(long startTime, long endTime) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryHistory> queryQueryHistoriesSubmitters(QueryHistoryRequest request, int size) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryStatistics> queryQueryHistoriesModelIds(QueryHistoryRequest request) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public QueryHistory getOldestQueryHistory(long index) {
+ return null;
+ }
+
+ @Override
+ public QueryHistory getOldestQueryHistory(String project, long index) {
+ return null;
+ }
+
+ @Override
+ public Long getCountOnQueryHistory() {
+ return 0L;
+ }
+
+ @Override
+ public Long getCountOnQueryHistory(long retainTime) {
+ return 0L;
+ }
+
+ @Override
+ public Map<String, Long> getCountGroupByProject() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public QueryHistory queryByQueryId(String queryId) {
+ return null;
+ }
+
+ @Override
+ public List<QueryHistory> queryAllQueryHistories() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryHistory> queryQueryHistoriesByIdOffset(long id, int batchSize, String project) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryStatistics> queryCountAndAvgDuration(long startTime, long endTime, String project) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryStatistics> queryCountByModel(long startTime, long endTime, String project) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public long queryQueryHistoryCountBeyondOffset(long offset, String project) {
+ return 0;
+ }
+
+ @Override
+ public long queryQueryHistoryMaxId(String project) {
+ return 0;
+ }
+
+ @Override
+ protected Long queryQueryHistoryMinQueryTime() {
+ return 0L;
+ }
+
+ @Override
+ public QueryStatistics queryRecentQueryCount(long startTime, long endTime, String project) {
+ return null;
+ }
+
+ @Override
+ public List<QueryStatistics> queryCountByTime(long startTime, long endTime, String timeDimension, String project) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryStatistics> queryAvgDurationByModel(long startTime, long endTime, String project) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<QueryStatistics> queryAvgDurationByTime(long startTime, long endTime, String timeDimension, String project) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void deleteQueryHistory() {
+ // nothing to do
+ }
+
+ @Override
+ public int deleteQueryHistory(long id) {
+ return 0;
+ }
+
+ @Override
+ public int deleteQueryHistory(String project, long id) {
+ return 0;
+ }
+
+ @Override
+ public void deleteQueryHistory(String project) {
+ // nothing to do
+ }
+
+ @Override
+ public void deleteQueryHistoryRealization(long queryTime) {
+ // nothing to do
+ }
+
+ @Override
+ public void deleteQueryHistoryRealization(String project, long queryTime) {
+ // nothing to do
+ }
+
+ @Override
+ public void deleteQueryHistoryRealization(String project) {
+ // nothing to do
+ }
+
+ @Override
+ public void updateQueryHistoryInfo(List<Pair<Long, QueryHistoryInfo>> idToQHInfoList) {
+ // nothing to do
+ }
+
+ @Override
+ InsertStatementProvider<QueryMetrics> getInsertQhProvider(QueryMetrics queryMetrics) {
+ return null;
+ }
+
+ @Override
+ InsertStatementProvider<RealizationMetrics> getInsertQhRealizationProvider(RealizationMetrics realizationMetrics) {
+ return null;
+ }
+
+ @Override
+ QueryExpressionDSL<SelectModel>.QueryExpressionWhereBuilder filterModelsByConditions(QueryExpressionDSL<SelectModel> selectSql, List<String> modelIds) {
+ return null;
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
index f70ff1ab7c..13d7aa6a90 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
@@ -64,7 +64,11 @@ public class RDBMSQueryHistoryDAO implements QueryHistoryDAO {
String metadataIdentifier = StorageURL.replaceUrl(config.getMetadataUrl());
this.queryMetricMeasurement = metadataIdentifier + "_" + QueryHistory.QUERY_MEASUREMENT_SURFIX;
this.realizationMetricMeasurement = metadataIdentifier + "_" + QueryHistory.REALIZATION_MEASUREMENT_SURFIX;
- jdbcQueryHisStore = new JdbcQueryHistoryStore(config);
+ if ("noop".equals(config.getQueryHistoryUrl().toString())) {
+ jdbcQueryHisStore = new NoopJdbcQueryHistoryStore();
+ } else {
+ jdbcQueryHisStore = new JdbcQueryHistoryStore(config);
+ }
}
@Override
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java
index 7435dc57ab..a5edaebdd0 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/SelectedColumnMeta.java
@@ -25,7 +25,6 @@ import org.apache.kylin.common.KylinConfig;
/**
*/
-@SuppressWarnings("serial")
public class SelectedColumnMeta implements Serializable {
public SelectedColumnMeta(boolean isAutoIncrement, boolean isCaseSensitive, boolean isSearchable,
boolean isCurrency, int isNullalbe, boolean isSigned, int displaySize, String label, String name,
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java
index bc3c151e05..b63f4ad852 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/SegmentControllerV2.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.controller.v2;
+import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
import static org.apache.kylin.common.exception.CommonErrorCode.FAILED_PARSE_JSON;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.CUBE_NOT_EXIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_DELETE_SELECT_EMPTY;
@@ -26,13 +27,13 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_NOT
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_REFRESH_MORE_THAN_ONE;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_REFRESH_SELECT_EMPTY;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_SELECT_EMPTY;
-import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
@@ -41,13 +42,14 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.job.execution.JobTypeEnum;
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TimeRange;
-import org.apache.kylin.rest.response.EnvelopeResponse;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.rest.controller.BaseController;
import org.apache.kylin.rest.request.CubeRebuildRequest;
import org.apache.kylin.rest.request.SegmentMgmtRequest;
+import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.JobInfoResponseV2;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.NDataModelResponse3X;
@@ -68,8 +70,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
+import org.apache.kylin.fileseg.FileSegments.ModelFileSegments;
import io.swagger.annotations.ApiOperation;
import lombok.val;
@@ -77,6 +78,8 @@ import lombok.val;
@RequestMapping(value = "/api/cubes", produces = { HTTP_VND_APACHE_KYLIN_V2_JSON })
public class SegmentControllerV2 extends BaseController {
+ private static final String ENABLE_FILE_SEGMENTS = "ENABLE_FILE_SEGMENTS";
+
@Autowired
@Qualifier("modelService")
private ModelService modelService;
@@ -88,14 +91,14 @@ public class SegmentControllerV2 extends BaseController {
@ApiOperation(value = "getCubes", tags = { "AI" })
@GetMapping(value = "")
@ResponseBody
- public EnvelopeResponse getCubes(@RequestParam(value = "projectName") String project,
+ public EnvelopeResponse<HashMap<String, Object>> getCubes(@RequestParam(value = "projectName") String project,
@RequestParam(value = "modelName", required = false) String modelAlias,
@RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer offset,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer limit) {
checkProjectName(project);
List<NDataModelResponse> modelsResponse = new ArrayList<>(modelService.getCubes(modelAlias, project));
- List<NDataModelResponse3X> result = Lists.newArrayList();
+ List<NDataModelResponse3X> result = new ArrayList<>();
try {
for (NDataModelResponse response : modelsResponse) {
result.add(NDataModelResponse3X.convert(response));
@@ -111,7 +114,7 @@ public class SegmentControllerV2 extends BaseController {
@ApiOperation(value = "getCube", tags = { "AI" })
@GetMapping(value = "/{cubeName}")
@ResponseBody
- public EnvelopeResponse getCube(@PathVariable("cubeName") String modelAlias,
+ public EnvelopeResponse<NDataModelResponse3X> getCube(@PathVariable("cubeName") String modelAlias,
@RequestParam(value = "project", required = false) String project) {
NDataModelResponse dataModelResponse = modelService.getCube(modelAlias, project);
if (Objects.isNull(dataModelResponse)) {
@@ -131,7 +134,7 @@ public class SegmentControllerV2 extends BaseController {
@ApiOperation(value = "rebuild", tags = { "DW" })
@PutMapping(value = "/{cubeName}/rebuild")
@ResponseBody
- public EnvelopeResponse rebuild(@PathVariable("cubeName") String modelAlias,
+ public EnvelopeResponse<JobInfoResponseV2> rebuild(@PathVariable("cubeName") String modelAlias,
@RequestParam(value = "project", required = false) String project, @RequestBody CubeRebuildRequest request)
throws Exception {
String startTime = String.valueOf(request.getStartTime());
@@ -183,21 +186,41 @@ public class SegmentControllerV2 extends BaseController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, result, "");
}
+ @ApiOperation(value = "getFileSegments", tags = { "DW" })
+ @GetMapping(value = "/{cubeName}/file_segments")
+ @ResponseBody
+ public EnvelopeResponse<?> getFileSegmentsStatus(@PathVariable("cubeName") String modelAlias,
+ @RequestParam(value = "project") String project) {
+ String projectChecked = checkProjectName(project);
+ ModelFileSegments ret = modelService.getModelFileSegments(projectChecked, modelAlias);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, ret, "done");
+ }
+
@ApiOperation(value = "manageSegments", tags = { "DW" })
@PutMapping(value = "/{cubeName}/segments")
@ResponseBody
- public EnvelopeResponse manageSegments(@PathVariable("cubeName") String modelAlias,
+ public EnvelopeResponse<?> manageSegments(@PathVariable("cubeName") String modelAlias,
@RequestParam(value = "project", required = false) String project,
@RequestBody SegmentMgmtRequest request) {
- if (CollectionUtils.isEmpty(request.getSegments())) {
- throw new KylinException(SEGMENT_SELECT_EMPTY);
- }
NDataModelResponse dataModelResponse = modelService.getCube(modelAlias, project);
if (Objects.isNull(dataModelResponse)) {
throw new KylinException(CUBE_NOT_EXIST);
}
+ if (ENABLE_FILE_SEGMENTS.equals(request.getBuildType())) {
+ Optional<List<String>> fileHashs = request.isForce() ? Optional.of(request.getSegments())
+ : Optional.empty();
+ modelService.forceFileSegments(request.getProject(), dataModelResponse.getId(), // model ID
+ request.getMpValues(), // storage location, like -- s3://maosha-public/liyang/cloud_billings
+ fileHashs, SegmentStatusEnum.NEW);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, "", "Enabled file segments successfully");
+ }
+
+ if (CollectionUtils.isEmpty(request.getSegments())) {
+ throw new KylinException(SEGMENT_SELECT_EMPTY);
+ }
+
List<NDataSegmentResponse> segList = dataModelResponse.getSegments().stream()
.filter(segment -> request.getSegments().contains(segment.getName())).collect(Collectors.toList());
Set<String> segNameSet = segList.stream().map(NDataSegmentResponse::getName).collect(Collectors.toSet());
@@ -221,8 +244,9 @@ public class SegmentControllerV2 extends BaseController {
if (CollectionUtils.isEmpty(idList)) {
throw new KylinException(SEGMENT_REFRESH_SELECT_EMPTY);
}
- val refreshResponse = modelBuildService.refreshSegmentById(new RefreshSegmentParams(
- dataModelResponse.getProject(), dataModelResponse.getId(), idList.toArray(new String[0])));
+ val refreshResponse = modelBuildService
+ .refreshSegmentById(new RefreshSegmentParams(dataModelResponse.getProject(),
+ dataModelResponse.getId(), idList.toArray(new String[0]), request.isForce()));
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, JobInfoResponseV2.convert(refreshResponse), "");
case "DROP":
if (CollectionUtils.isEmpty(idList)) {
@@ -239,14 +263,14 @@ public class SegmentControllerV2 extends BaseController {
@ApiOperation(value = "getHoles", tags = { "DW" })
@GetMapping(value = "/{cubeName}/holes")
@ResponseBody
- public EnvelopeResponse getHoles(@PathVariable("cubeName") String modelAlias,
+ public EnvelopeResponse<List<NDataSegment>> getHoles(@PathVariable("cubeName") String modelAlias,
@RequestParam(value = "project", required = false) String project) {
NDataModelResponse dataModelResponse = modelService.getCube(modelAlias, project);
if (Objects.isNull(dataModelResponse)) {
throw new KylinException(CUBE_NOT_EXIST);
}
- List<NDataSegment> holes = Lists.newArrayList();
+ List<NDataSegment> holes = new ArrayList<>();
List<NDataSegmentResponse> segments = dataModelResponse.getSegments();
Collections.sort(segments);
@@ -272,7 +296,7 @@ public class SegmentControllerV2 extends BaseController {
@ApiOperation(value = "getSql", tags = { "AI" })
@GetMapping(value = "/{cubeName}/sql")
@ResponseBody
- public EnvelopeResponse getSql(@PathVariable("cubeName") String modelAlias,
+ public EnvelopeResponse<Properties> getSql(@PathVariable("cubeName") String modelAlias,
@RequestParam(value = "project", required = false) String project) {
NDataModelResponse dataModelResponse = modelService.getCube(modelAlias, project);
if (Objects.isNull(dataModelResponse)) {
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
index dd4afcb0f7..d0cd20827d 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
@@ -371,16 +371,24 @@ public class ModelBuildService extends AbstractModelService implements ModelBuil
TableDesc table = getManager(NTableMetadataManager.class, project)
.getTableDesc(modelDescInTransaction.getRootFactTableName());
val df = getManager(NDataflowManager.class, project).getDataflow(modelId);
- if (modelDescInTransaction.getPartitionDesc() == null
- || StringUtils.isEmpty(modelDescInTransaction.getPartitionDesc().getPartitionDateColumn())) {
- throw new IllegalArgumentException("Can not add a new segment on full build model.");
- }
- Preconditions.checkArgument(!PushDownUtil.needPushdown(params.getStart(), params.getEnd()),
- "Load data must set start and end date");
- val segmentRangeToBuild = SourceFactory.getSource(table).getSegmentRange(params.getStart(), params.getEnd());
- List<NDataSegment> overlapSegments = modelService.checkSegmentToBuildOverlapsBuilt(project,
- modelDescInTransaction, segmentRangeToBuild, params.isNeedBuild(), params.getBatchIndexIds());
- buildSegmentOverlapExceptionInfo(overlapSegments);
+
+ var segmentRangeToBuild = params.getSpecifiedSegmentRange();
+ if (segmentRangeToBuild == null) {
+ if (modelDescInTransaction.getPartitionDesc() == null
+ || StringUtils.isEmpty(modelDescInTransaction.getPartitionDesc().getPartitionDateColumn())) {
+ throw new IllegalArgumentException("Can not add a new segment on full build model.");
+ }
+ Preconditions.checkArgument(!PushDownUtil.needPushdown(params.getStart(), params.getEnd()),
+ "Load data must set start and end date");
+ segmentRangeToBuild = SourceFactory.getSource(table).getSegmentRange(params.getStart(), params.getEnd());
+ if (segmentRangeToBuild instanceof SegmentRange.BasicSegmentRange) {
+ List<NDataSegment> overlapSegments = modelService.checkSegmentToBuildOverlapsBuilt(project,
+ modelDescInTransaction, (SegmentRange.BasicSegmentRange) segmentRangeToBuild,
+ params.isNeedBuild(), params.getBatchIndexIds());
+ buildSegmentOverlapExceptionInfo(overlapSegments);
+ }
+ }
+
modelService.saveDateFormatIfNotExist(project, modelId, params.getPartitionColFormat());
checkMultiPartitionBuildParam(modelDescInTransaction, params);
NDataSegment newSegment = getManager(NDataflowManager.class, project).appendSegment(df, segmentRangeToBuild,
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index e542fe6698..ba699d6c8d 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -89,6 +89,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
@@ -276,6 +277,8 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.kylin.fileseg.FileSegments;
+import org.apache.kylin.fileseg.FileSegments.ModelFileSegments;
import io.kyligence.kap.secondstorage.SecondStorage;
import io.kyligence.kap.secondstorage.SecondStorageNodeHelper;
import io.kyligence.kap.secondstorage.SecondStorageUpdater;
@@ -373,7 +376,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
private void addOldSegmentParams(NDataModel model, NDataModelOldParams oldParams,
List<AbstractExecutable> executables) {
List<NDataSegmentResponse> segments = getSegmentsResponse(model.getId(), model.getProject(), "1",
- String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true, (model instanceof NDataModelLiteResponse));
+ String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true,
+ (model instanceof NDataModelLiteResponse));
calculateRecordSizeAndCount(segments, oldParams);
if (model instanceof NDataModelResponse) {
((NDataModelResponse) model).setSegments(segments);
@@ -2865,6 +2869,41 @@ public class ModelService extends AbstractModelService implements TableModelSupp
offlineModelIfNecessary(dataflowManager, model);
}
+ public ModelFileSegments getModelFileSegments(String project, String modelAlias) {
+ aclEvaluate.checkProjectOperationPermission(project);
+ NDataflowManager dfManager = getManager(NDataflowManager.class, project);
+ NDataflow df = dfManager.getDataflowByModelAlias(modelAlias);
+ if (df == null || df.isBroken() || df.getModel().isBroken())
+ return ModelFileSegments.broken(project, modelAlias);
+
+ return FileSegments.getModelFileSegments(project, df.getModel().getId(), true);
+ }
+
+ @Transaction(project = 0)
+ public void forceFileSegments(String project, String modelId, String storageLocation,
+ Optional<List<String>> fileHashs, SegmentStatusEnum initStatus) {
+ aclEvaluate.checkProjectOperationPermission(project);
+ NDataflowManager dfManager = getManager(NDataflowManager.class, project);
+ FileSegments.forceFileSegments(project, modelId, storageLocation, fileHashs, (fileSegRangeToCreate) -> {
+ NDataSegment newSeg;
+ if (initStatus == SegmentStatusEnum.NEW) {
+ // normal new segment and kickoff build immediately
+ IncrementBuildSegmentParams incrParams = new IncrementBuildSegmentParams(project, modelId,
+ fileSegRangeToCreate, true);
+ modelBuildService.constructIncrementBuild(incrParams);
+ newSeg = dfManager.getDataflow(modelId).getSegmentByName(fileSegRangeToCreate.proposeSegmentName());
+ } else {
+ // a new READY segment without immediate build
+ newSeg = dfManager.appendSegment(dfManager.getDataflow(modelId), fileSegRangeToCreate, initStatus,
+ null);
+ }
+ return newSeg;
+ }, (segmentIdToDelete) -> {
+ deleteSegmentById(modelId, project, new String[] { segmentIdToDelete }, true);
+ return true;
+ });
+ }
+
private void offlineModelIfNecessary(NDataflowManager dfManager, String modelId) {
NDataflow df = dfManager.getDataflow(modelId);
if (df.getSegments().isEmpty() && RealizationStatusEnum.ONLINE == df.getStatus()) {
@@ -3598,7 +3637,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
validateFusionModelDimension(modelRequest);
NDataModel model = semanticUpdater.convertToDataModel(modelRequest);
- for(NDataModel.Measure measure : model.getAllMeasures()) {
+ for (NDataModel.Measure measure : model.getAllMeasures()) {
measure.getFunction().init(model);
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IncrementBuildSegmentParams.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IncrementBuildSegmentParams.java
index 06a7082da4..bfa6bb7289 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IncrementBuildSegmentParams.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/IncrementBuildSegmentParams.java
@@ -22,14 +22,14 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.rest.request.SegmentTimeRequest;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@@ -38,6 +38,7 @@ import lombok.Setter;
@Getter
@Setter
public class IncrementBuildSegmentParams extends FullBuildSegmentParams {
+ private SegmentRange<?> specifiedSegmentRange;
private String start;
private String end;
private PartitionDesc partitionDesc;
@@ -47,6 +48,11 @@ public class IncrementBuildSegmentParams extends FullBuildSegmentParams {
private List<String[]> multiPartitionValues;
private boolean buildAllSubPartitions;
+ public IncrementBuildSegmentParams(String project, String modelId, SegmentRange<?> specifiedSegmentRange, boolean needBuild) {
+ super(project, modelId, needBuild);
+ this.specifiedSegmentRange = specifiedSegmentRange;
+ }
+
public IncrementBuildSegmentParams(String project, String modelId, String start, String end,
String partitionColFormat, boolean needBuild, List<String[]> multiPartitionValues) {
super(project, modelId, needBuild);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 5835ab21b1..8e5dce5055 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -30,6 +30,9 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.kylin.fileseg.FileSegments;
+import io.kyligence.kap.secondstorage.SecondStorageUtil;
+
import org.apache.calcite.rel.AbstractRelNode;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableScan;
@@ -72,7 +75,6 @@ import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.Getter;
import lombok.Setter;
import lombok.val;
@@ -304,8 +306,19 @@ public class OLAPContext {
realization.setCxtId(ctx.id);
realization.setSecondStorage(
QueryContext.current().getSecondStorageUsageMap().getOrDefault(realization.getLayoutId(), false));
- realization.setRecommendSecondStorage(
- recommendSecondStorage(ctx.realization.getProject(), modelId, realizationType));
+ realization.setRecommendSecondStorage(recommendSecondStorage(ctx.realization.getProject(), modelId, realizationType));
+
+ // lastDataLoadTime & isLoadingData
+ if (ctx.realization instanceof NDataflow) {
+ NDataflow df = (NDataflow) ctx.realization;
+ if (df.getModel().isFilePartitioned()) {
+ boolean isLoadingData = df.getSegments().stream().anyMatch(seg -> seg.getStatus() == SegmentStatusEnum.NEW);
+ realization.setLoadingData(isLoadingData);
+ realization.setBuildingIndex(FileSegments.guessIsBuildingIndex(df));
+ realization.setLastDataRefreshTime(df.getLastDataRefreshTime());
+ }
+ }
+
return realization;
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryLayoutChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryLayoutChooser.java
index 3a0d29fa4e..5db653ba0e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryLayoutChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/QueryLayoutChooser.java
@@ -46,6 +46,7 @@ import org.apache.kylin.metadata.cube.model.NDataLayout;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.model.DeriveInfo;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.realization.CapabilityResult;
@@ -252,10 +253,14 @@ public class QueryLayoutChooser {
return commonLayouts.values();
}
+ val isMppOnTheFlyLayoutsEnabled = dataflow.getConfig().isMppOnTheFlyLayoutsEnabled();
+
String segmentOnlineMode = projectConfig.getKylinEngineSegmentOnlineMode();
for (int i = 0; i < segments.size(); i++) {
val dataSegment = segments.get(i);
var layoutIdMapToDataLayout = dataSegment.getLayoutsMap();
+ val isReadyEmptySeg = isMppOnTheFlyLayoutsEnabled
+ && dataSegment.getStatus() == SegmentStatusEnum.READY && layoutIdMapToDataLayout.isEmpty();
if (SegmentOnlineMode.ANY.toString().equalsIgnoreCase(segmentOnlineMode)
&& MapUtils.isNotEmpty(chSegmentToLayoutsMap)) {
// Only the basic TableIndex is built in the CH storage
@@ -267,7 +272,7 @@ public class QueryLayoutChooser {
nDataLayoutMap.putAll(layoutIdMapToDataLayout);
layoutIdMapToDataLayout = nDataLayoutMap;
}
- if (i == 0) {
+ if (i == 0 || isReadyEmptySeg) { // ACCEPT empty & ready segments, they can be served by mpp on-the-fly
commonLayouts.putAll(layoutIdMapToDataLayout);
} else {
commonLayouts.keySet().retainAll(layoutIdMapToDataLayout.keySet());
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 9cb44c6736..3757d717d8 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -66,6 +66,7 @@ import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.metadata.realization.RoutingIndicatorException;
import org.apache.kylin.query.exception.NoAuthorizedColsError;
import org.apache.kylin.query.security.AccessDeniedException;
@@ -117,9 +118,14 @@ public class PushDownUtil {
return null;
}
if (queryParams.isSelect()) {
- logger.info("Query:[{}] failed to utilize pre-calculation, routing to other engines",
- QueryContext.current().getMetrics().getCorrectedSql(), queryParams.getSqlException());
- if (!queryParams.isForcedToPushDown() && !isExpectedCause(queryParams.getSqlException())) {
+ SQLException sqlEx = queryParams.getSqlException();
+ if (ExceptionUtils.getRootCause(sqlEx) instanceof NoRealizationFoundException)
+ logger.info("Query has no match model, routing to other engines -- {}",
+ QueryContext.current().getMetrics().getCorrectedSql());
+ else
+ logger.info("Query hit error utilizing pre-calculation, routing to other engines -- {}",
+ QueryContext.current().getMetrics().getCorrectedSql(), sqlEx);
+ if (!queryParams.isForcedToPushDown() && !isExpectedCause(sqlEx)) {
logger.info("quit doPushDownQuery because prior exception thrown is unexpected");
return null;
}
@@ -145,6 +151,7 @@ public class PushDownUtil {
// on no authorized cols found, return empty result
return PushdownResult.emptyResult();
}
+ QueryContext.current().record("massage");
QueryContext.currentTrace().startSpan(QueryTrace.PREPARE_AND_SUBMIT_JOB);
if (queryParams.isSelect()) {
@@ -492,7 +499,7 @@ public class PushDownUtil {
return true;
}
- return false;
+ return QueryContext.current().getQueryTagInfo().isWithoutSyntaxError();
}
public static String calcStart(String start, SegmentRange<?> coveredRange) {
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index 9f16d94e68..6d69ea475f 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -33,9 +33,12 @@ import java.text.SimpleDateFormat;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -57,6 +60,9 @@ import org.apache.kylin.common.persistence.transaction.StopQueryBroadcastEventNo
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryHistoryRequest;
import org.apache.kylin.metadata.query.util.QueryHisTransformStandardUtil;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -70,6 +76,7 @@ import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLFormatRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
+import org.apache.kylin.rest.request.SyncFileSegmentsRequest;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.QueryHistoryFiltersResponse;
@@ -77,6 +84,8 @@ import org.apache.kylin.rest.response.QueryStatisticsResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.response.ServerExtInfoResponse;
import org.apache.kylin.rest.response.ServerInfoResponse;
+import org.apache.kylin.rest.response.SyncFileSegmentsResponse;
+import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.QueryCacheManager;
import org.apache.kylin.rest.service.QueryHistoryService;
import org.apache.kylin.rest.service.QueryService;
@@ -103,6 +112,8 @@ import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
+import org.apache.kylin.fileseg.FileSegments;
+import org.apache.kylin.fileseg.FileSegmentsDetector;
import io.swagger.annotations.ApiOperation;
import lombok.val;
import redis.clients.jedis.exceptions.JedisException;
@@ -135,6 +146,9 @@ public class NQueryController extends NBasicController {
@Autowired
private AclEvaluate aclEvaluate;
+ @Autowired
+ private ModelService modelService;
+
@Override
protected Logger getLogger() {
return logger;
@@ -179,10 +193,83 @@ public class NQueryController extends NBasicController {
checkProjectName(sqlRequest.getProject());
sqlRequest.setUserAgent(userAgent != null ? userAgent : "");
QueryContext.current().record("end_http_proc");
- SQLResponse sqlResponse = queryService.queryWithCache(sqlRequest);
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, sqlResponse, "");
+
+ // take chance of push-down to detect and apply file segment changes
+ boolean detectFileSegments = sqlRequest.isForcedToPushDown();
+ if (detectFileSegments)
+ FileSegmentsDetector.startLocally(sqlRequest.getProject());
+
+ try {
+ SQLResponse sqlResponse = queryService.queryWithCache(sqlRequest);
+
+ if (detectFileSegments) {
+ FileSegmentsDetector.report(finding -> {
+ if (FileSegments.isSyncFileSegSql(sqlRequest.getSql())) {
+ // return modelIds to syncFileSegments()
+ sqlResponse.addNativeRealizationIfNotExist(finding.modelId);
+ }
+ modelService.forceFileSegments(finding.project, finding.modelId,
+ finding.storageLocation, Optional.of(finding.fileHashs), SegmentStatusEnum.NEW);
+ });
+ }
+
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, sqlResponse, "");
+
+ } finally {
+ if (detectFileSegments)
+ FileSegmentsDetector.endLocally();
+ }
+ }
+
+ @PostMapping(value = "/sync_file_segments")
+ @ResponseBody
+ public EnvelopeResponse<SyncFileSegmentsResponse> syncFileSegments(@RequestBody SyncFileSegmentsRequest req) {
+ String project = req.getProject();
+ checkProjectName(project);
+
+ String inputSql = req.getSql();
+ List<NDataModel> inputModels = FileSegments.findModelsOfFileSeg(project, req.getModelAliasOrFactTables());
+
+ StringBuilder probeSql = new StringBuilder();
+ if (!StringUtils.isBlank(inputSql)) {
+ probeSql.append(FileSegments.makeSyncFileSegSql(inputSql));
+ }
+ if (inputModels.size() > 0) {
+ if (probeSql.length() > 0)
+ probeSql.append("\n union all \n");
+
+ probeSql.append(FileSegments.makeSyncFileSegSql(inputModels));
+ }
+ if (probeSql.length() == 0) {
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, null, "");
+ }
+
+ PrepareSqlRequest sqlRequest = new PrepareSqlRequest();
+ sqlRequest.setProject(project);
+ sqlRequest.setForcedToPushDown(true);
+ sqlRequest.setSql(probeSql.toString());
+
+ EnvelopeResponse<SQLResponse> probeResponse = query(sqlRequest, "");
+
+ Set<String> touchedModelIds = new HashSet<>();
+ for (NDataModel inputModel : inputModels) {
+ touchedModelIds.add(inputModel.getId());
+ }
+ if (probeResponse.getData() != null && probeResponse.getData().getNativeRealizations() != null) {
+ for (NativeQueryRealization real : probeResponse.getData().getNativeRealizations()) {
+ touchedModelIds.add(real.getModelId());
+ }
+ }
+
+ SyncFileSegmentsResponse resp = new SyncFileSegmentsResponse();
+ resp.setProject(project);
+ resp.setModels(touchedModelIds.stream()
+ .map(modelId -> FileSegments.getModelFileSegments(project, modelId))
+ .collect(Collectors.toList()));
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, resp, "");
}
+
@ApiOperation(value = "cancelQuery", tags = { "QE" })
@DeleteMapping(value = "/{id:.+}")
@ResponseBody
@@ -442,7 +529,7 @@ public class NQueryController extends NBasicController {
@ApiOperation(value = "getServers", tags = { "QE" })
@GetMapping(value = "/servers")
@ResponseBody
- public EnvelopeResponse<List> getServers(
+ public EnvelopeResponse<List<?>> getServers(
@RequestParam(value = "ext", required = false, defaultValue = "false") boolean ext) {
if (ext) {
List<ServerExtInfoResponse> serverInfo =
@@ -470,10 +557,7 @@ public class NQueryController extends NBasicController {
if (StringUtils.isEmpty(param1) && StringUtils.isEmpty(param2))
return true;
- if (StringUtils.isNotEmpty(param1) && StringUtils.isNotEmpty(param2))
- return true;
-
- return false;
+ return StringUtils.isNotEmpty(param1) && StringUtils.isNotEmpty(param2);
}
@PostMapping(value = "/format/{format:.+}", consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
@@ -526,7 +610,7 @@ public class NQueryController extends NBasicController {
logger.error("Download query result failed...", e);
throw new InternalErrorException(e);
} finally {
- IOUtils.closeQuietly(csvWriter);
+ IOUtils.closeQuietly(csvWriter, null);
}
}
diff --git a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
index d6e96b7baf..0990bee1fa 100644
--- a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
+++ b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
@@ -50,6 +50,7 @@ import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.QueryHistoryFiltersResponse;
+import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.response.ServerInfoResponse;
import org.apache.kylin.rest.service.QueryCacheManager;
import org.apache.kylin.rest.service.QueryHistoryService;
@@ -610,4 +611,30 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
.param("ext", "true").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
}
+
+ @Test
+ public void testSqlResponseAddNativeRealization() {
+ final String modelId = "m123";
+
+ // add native realization
+ {
+ SQLResponse res = new SQLResponse();
+ res.addNativeRealizationIfNotExist(modelId);
+ List<NativeQueryRealization> reals = res.getNativeRealizations();
+ Assert.assertEquals(1, reals.size());
+ Assert.assertEquals(modelId, reals.get(0).getModelId());
+ }
+
+ // native realization already exists
+ {
+ SQLResponse res = new SQLResponse();
+ List<NativeQueryRealization> reals = new ArrayList<>();
+ reals.add(new NativeQueryRealization(modelId, null, null));
+ res.setNativeRealizations(reals);
+ res.addNativeRealizationIfNotExist(modelId);
+ List<NativeQueryRealization> realsGot = res.getNativeRealizations();
+ Assert.assertEquals(1, realsGot.size());
+ Assert.assertEquals(modelId, realsGot.get(0).getModelId());
+ }
+ }
}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index bbee32c7dc..0d9d22f316 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.response;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -100,6 +101,22 @@ public class SQLResponse implements Serializable {
protected String storageCacheType;
+ /**
+ * The data fetching time of the datasets that contribute to the query result. The time stands for the data
+ * freshness of the query result.
+ * <ul>
+ * <li>If 'dataFetchTime' = now(), meaning the query gets most fresh data from source, typically executed
+ * by a pushdown engine, and bypasses all caches along the way, including query cache, cube segment cache,
+ * soft affinity local cache etc..</li>
+ * <li>Or otherwise, 'dataFetchTime' should be set to the cache build time, the time when the data is last
+ * fetched from source and the cache is built.</li>
+ * <li>If multiple datasets and multiple data caches are involved in a query, the LATEST data fetch time
+ * among them is returned.</li>
+ * </ul>
+ */
+ @JsonProperty("dataFetchTime")
+ protected long dataFetchTime;
+
@JsonProperty("pushDown")
protected boolean queryPushDown = false;
@@ -191,9 +208,7 @@ public class SQLResponse implements Serializable {
this.setQueryId(queryContext.getQueryId());
this.setScanRows(queryContext.getMetrics().getScanRows());
this.setScanBytes(queryContext.getMetrics().getScanBytes());
-
this.setShufflePartitions(queryContext.getShufflePartitions());
-
return this;
}
@@ -214,4 +229,29 @@ public class SQLResponse implements Serializable {
results = ImmutableList.copyOf(results);
}
}
+
+ public void updateDataFetchTime(QueryContext queryContext) {
+ // The dataFetchTime could come from:
+ // 1) from queryContext.getMetrics().getDataFetchTime(), in case of push-down, this is the time from local file cache (KylinCacheFileSystem)
+ // 2) from this.nativeRealizations, in case of index query, this is the time when segment is built
+ // We take the max of the above all.
+
+ dataFetchTime = queryContext.getMetrics().getDataFetchTime();
+
+ if (nativeRealizations != null) {
+ for (NativeQueryRealization real : nativeRealizations) {
+ dataFetchTime = Math.max(dataFetchTime, real.getLastDataRefreshTime());
+ }
+ }
+ }
+
+ public void addNativeRealizationIfNotExist(String modelId) {
+ if (nativeRealizations == null)
+ nativeRealizations = new ArrayList<>();
+
+ if (nativeRealizations.stream().anyMatch(real -> real.getModelId().equals(modelId)))
+ return;
+
+ nativeRealizations.add(new NativeQueryRealization(modelId, null, null));
+ }
}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java b/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java
index 33d9332cbd..42e20ece84 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/response/SQLResponseV2.java
@@ -88,6 +88,24 @@ public class SQLResponseV2 implements Serializable {
protected String storageCacheType;
+ /**
+ * The data fetching time of the datasets that contribute to the query result. The time stands for the data
+ * freshness of the query result.
+ * <ul>
+ * <li>If 'dataFetchTime' = now(), meaning the query gets most fresh data from source, typically executed
+ * by a pushdown engine, and bypasses all caches along the way, including query cache, cube segment cache,
+ * soft affinity local cache etc..</li>
+ * <li>Or otherwise, 'dataFetchTime' should be set to the cache build time, the time when the data is last
+ * fetched from source and the cache is built.</li>
+ * <li>If multiple datasets and multiple data caches are involved in a query, the LATEST data fetch time
+ * among them is returned.</li>
+ * TODO: As of March 2023, only the spark-sql pushdown engine reports its alluxio local cache time (soft affinity);
+ * The cache time of query cache and cube cache is yet to be implemented.
+ * </ul>
+ */
+ @JsonProperty("dataFetchTime")
+ protected long dataFetchTime;
+
@JsonProperty("pushDown")
protected boolean queryPushDown = false;
@@ -144,6 +162,7 @@ public class SQLResponseV2 implements Serializable {
this.hitExceptionCache = sqlResponse.isHitExceptionCache();
this.storageCacheUsed = sqlResponse.isStorageCacheUsed();
this.storageCacheType = sqlResponse.getStorageCacheType();
+ this.dataFetchTime = sqlResponse.getDataFetchTime();
this.queryPushDown = sqlResponse.isQueryPushDown();
this.isPrepare = sqlResponse.isPrepare();
this.isTimeout = sqlResponse.isTimeout();
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 3d453ce642..7b6af176aa 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -442,6 +442,9 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
BloomFilterSkipCollector.logAndCleanStatus(QueryContext.current().getQueryId());
ParquetPageFilterCollector.logParquetPages(QueryContext.current().getQueryId());
+
+ QueryContext.current().record("end");
+
LogReport report = new LogReport().put(LogReport.QUERY_ID, QueryContext.current().getQueryId())
.put(LogReport.SQL, sql).put(LogReport.USER, user)
.put(LogReport.SUCCESS, null == response.getExceptionMessage()).put(LogReport.DURATION, duration)
@@ -459,6 +462,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
.put(LogReport.HIT_EXCEPTION_CACHE, response.isHitExceptionCache())
.put(LogReport.STORAGE_CACHE_USED, response.isStorageCacheUsed())
.put(LogReport.STORAGE_CACHE_TYPE, response.getStorageCacheType())
+ .put(LogReport.DATA_FETCH_TIME, response.getDataFetchTime())
.put(LogReport.PUSH_DOWN, response.isQueryPushDown()).put(LogReport.IS_PREPARE, response.isPrepare())
.put(LogReport.TIMEOUT, response.isTimeout())
.put(LogReport.TIMELINE_SCHEMA, QueryContext.current().getSchema())
@@ -617,7 +621,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
sqlResponse = searchCache(sqlRequest, kylinConfig);
}
- // real execution if required
+ // REAL EXECUTION, if required
if (sqlResponse == null) {
try (QueryRequestLimits ignored = new QueryRequestLimits(project)) {
sqlResponse = queryAndUpdateCache(sqlRequest, kylinConfig);
@@ -1369,8 +1373,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
QueryContext queryContext = QueryContext.current();
response.wrapResultOfQueryContext(queryContext);
-
response.setNativeRealizations(OLAPContext.getNativeRealizations());
+ response.updateDataFetchTime(queryContext);
if (!queryContext.getQueryTagInfo().isVacant()) {
setAppMaterURL(response);
@@ -1496,6 +1500,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
static final String HIT_EXCEPTION_CACHE = "hit_exception_cache";
static final String STORAGE_CACHE_USED = "storage_cache_used";
static final String STORAGE_CACHE_TYPE = "storage_cache_type";
+ static final String DATA_FETCH_TIME = "data_fetch_time";
static final String PUSH_DOWN = "push_down";
static final String IS_PREPARE = "is_prepare";
static final String TIMEOUT = "timeout";
@@ -1523,6 +1528,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
.put(SHUFFLE_PARTITIONS, "Shuffle partitions: ").put(ACCEPT_PARTIAL, "Accept Partial: ")
.put(PARTIAL_RESULT, "Is Partial Result: ").put(HIT_EXCEPTION_CACHE, "Hit Exception Cache: ")
.put(STORAGE_CACHE_USED, "Storage Cache Used: ").put(STORAGE_CACHE_TYPE, "Storage Cache Type: ")
+ .put(DATA_FETCH_TIME, "Data Fetch Time: ")
.put(PUSH_DOWN, "Is Query Push-Down: ").put(IS_PREPARE, "Is Prepare: ").put(TIMEOUT, "Is Timeout: ")
.put(TRACE_URL, "Trace URL: ").put(TIMELINE_SCHEMA, "Time Line Schema: ").put(TIMELINE, "Time Line: ")
.put(ERROR_MSG, "Message: ").put(USER_TAG, "User Defined Tag: ")
@@ -1579,6 +1585,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
+ O2N.get(HIT_EXCEPTION_CACHE) + get(HIT_EXCEPTION_CACHE) + newLine //
+ O2N.get(STORAGE_CACHE_USED) + get(STORAGE_CACHE_USED) + newLine //
+ O2N.get(STORAGE_CACHE_TYPE) + get(STORAGE_CACHE_TYPE) + newLine //
+ + O2N.get(DATA_FETCH_TIME) + get(DATA_FETCH_TIME) + newLine //
+ O2N.get(PUSH_DOWN) + get(PUSH_DOWN) + newLine //
+ O2N.get(IS_PREPARE) + get(IS_PREPARE) + newLine //
+ O2N.get(TIMEOUT) + get(TIMEOUT) + newLine //
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index dc0572b6f8..d9015fb340 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -46,12 +46,14 @@ import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.cache.kylin.KylinCacheFileSystem;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryTrace;
import org.apache.kylin.common.ReadFsSwitch;
import org.apache.kylin.common.exception.DryRunSucceedException;
+import org.apache.kylin.fileseg.FileSegments;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -175,6 +177,10 @@ public class QueryExec {
QueryContext.current().setDryRun(true);
}
beforeQuery();
+
+ // process cache hint like -- select * from xxx /*+ ACCEPT_CACHE_TIME(158176387682000) */
+ sql = processAcceptCacheTimeInSql(sql);
+
QueryContext.currentTrace().startSpan(QueryTrace.SQL_PARSE_AND_OPTIMIZE);
RelRoot relRoot = sqlConverter.convertSqlToRelNode(sql);
queryContext.record("end_convert_to_relnode");
@@ -338,6 +344,8 @@ public class QueryExec {
private void afterQuery() {
Prepare.CatalogReader.THREAD_LOCAL.remove();
KECalciteConfig.THREAD_LOCAL.remove();
+ FileSegments.clearFileSegFilterLocally();
+ clearAcceptCacheTimeLocally();
}
public void setContextVar(String name, Object val) {
@@ -520,11 +528,11 @@ public class QueryExec {
String diagnosticStr = "";
String plan = " not exists";
boolean dryRun = QueryContext.current().isDryRun();
- if(e instanceof DryRunSucceedException){
- DryRunSucceedException d = (DryRunSucceedException)e;
+ if (e instanceof DryRunSucceedException) {
+ DryRunSucceedException d = (DryRunSucceedException) e;
plan = d.plan();
}
- if(dryRun) {
+ if (dryRun) {
StringBuilder diagnosticInfo = new StringBuilder(System.getProperty("line.separator"));
diagnosticInfo.append("<-------------------- Dry Run Info -------------------->");
@@ -540,7 +548,7 @@ public class QueryExec {
diagnosticInfo.append(SEP)
.append("3. OLAPContext(s) and matched model(s) :");
- if(OLAPContext.getThreadLocalContexts() != null) {
+ if (OLAPContext.getThreadLocalContexts() != null) {
String olapMatchInfo =
OLAPContext.getNativeRealizations().stream()
.map(r -> String.format(" Ctx=%d, \tMatched=%s, \tIndexType=%s, \tLayoutId=%d",
@@ -555,7 +563,7 @@ public class QueryExec {
for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
diagnosticInfo.append(SEP);
diagnosticInfo.append(" Ctx=").append(ctx.id);
- if(ctx.realization == null) {
+ if (ctx.realization == null) {
diagnosticInfo.append(" is not matched by any model/snapshot, expected ")
.append(ctx.tipsForUser());
} else {
@@ -589,6 +597,19 @@ public class QueryExec {
}
}
+ private String processAcceptCacheTimeInSql(String sql) {
+ if (kylinConfig.isKylinLocalCacheEnabled() && kylinConfig.isKylinFileStatusCacheEnabled()) {
+ return KylinCacheFileSystem.processAcceptCacheTimeInSql(sql);
+ }
+ return sql;
+ }
+
+ private void clearAcceptCacheTimeLocally() {
+ if (kylinConfig.isKylinLocalCacheEnabled() && kylinConfig.isKylinFileStatusCacheEnabled()) {
+ KylinCacheFileSystem.clearAcceptCacheTimeLocally();
+ }
+ }
+
public final static String SEP = System.getProperty("line.separator");
public final static String DRY_RUN_TIP = SEP
+ "Tip : Dryrun is a experimental feature for user to create proper model."
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/exec/calcite/CalciteQueryPlanExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/exec/calcite/CalciteQueryPlanExec.java
index 3d03f5772e..c3ba4e614f 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/exec/calcite/CalciteQueryPlanExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/exec/calcite/CalciteQueryPlanExec.java
@@ -97,6 +97,9 @@ public class CalciteQueryPlanExec implements QueryPlanExec {
// may induce some puzzle result
private String rawQueryResultToString(Object object, RelDataType dataType) {
String value = String.valueOf(object);
+ if (object == null) {
+ return value;
+ }
switch (dataType.getSqlTypeName()) {
case DATE:
return DateFormat.formatDayToEpochToDateStr(Long.parseLong(value), TimeZone.getTimeZone("GMT"));
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
index b6620a0091..ae748fa863 100644
--- a/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
+++ b/src/query/src/main/java/org/apache/kylin/query/util/QueryContextCutter.java
@@ -59,7 +59,7 @@ public class QueryContextCutter {
}
/**
- * For each query parse tree, the following steps are used for generating OlapContexts
+ * For each query parse tree, the following steps are used for generating OlapContexts
* and matching the precomputed indexes.
* <p> 1. The larger the OlapContext for the first cut, the better;</p>
* <p> 2. Traverse the RelNode operator tree to collect all attributes for each split OlapContext;</p>
diff --git a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
index 3e41fa541d..39aab3b023 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
@@ -282,6 +282,10 @@ public class QueryNodeFilter extends BaseFilter {
}
private boolean checkProcessLocal(KylinConfig kylinConfig, String project, String contentType) {
+ if (kylinConfig.isDevOrUT()) {
+ return true;
+ }
+
if (kylinConfig.isQueryNodeOnly()) {
return false;
}
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index 9d9af5980f..1f1d359eb6 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -44,7 +44,6 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -78,7 +77,6 @@ import org.apache.kylin.engine.spark.utils.HDFSUtils;
import org.apache.kylin.engine.spark.utils.JobMetricsUtils;
import org.apache.kylin.engine.spark.utils.SparkConfHelper;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.model.NDataModel;
@@ -103,6 +101,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTableType;
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
import org.apache.spark.sql.catalyst.rules.Rule;
import org.apache.spark.sql.execution.datasource.AlignmentTableStats;
+import org.apache.spark.sql.hive.HiveStorageRule;
import org.apache.spark.sql.hive.utils.ResourceDetectUtils;
import org.apache.spark.util.Utils;
import org.slf4j.Logger;
@@ -116,7 +115,7 @@ import scala.runtime.BoxedUnit;
public abstract class SparkApplication implements Application {
private static final Logger logger = LoggerFactory.getLogger(SparkApplication.class);
- private Map<String, String> params = Maps.newHashMap();
+ private Map<String, String> params = new HashMap<>();
public static final String JOB_NAME_PREFIX = "job_step_";
private static final String JOB_ERROR_API = "/kylin/api/jobs/error";
protected IJobProgressReport report;
@@ -218,7 +217,6 @@ public abstract class SparkApplication implements Application {
* get tracking url by application id
*
* @param sparkSession build sparkSession
- * @return
*/
public String getTrackingUrl(IClusterManager clusterManager, SparkSession sparkSession) {
return clusterManager.getBuildTrackingUrl(sparkSession);
@@ -296,7 +294,7 @@ public abstract class SparkApplication implements Application {
monitorSparkMaster();
- HadoopUtil.setCurrentConfiguration(new Configuration());
+ HadoopUtil.setCurrentConfiguration(HadoopUtil.getHadoopConfFromSparkEngine());
////////
exchangeSparkConf(buildEnv.sparkConf());
@@ -393,6 +391,14 @@ public abstract class SparkApplication implements Application {
return new AlignmentTableStats(session);
}
});
+ v1.injectPostHocResolutionRule(new AbstractFunction1<SparkSession, Rule<LogicalPlan>>() {
+ @Override
+ public Rule<LogicalPlan> apply(SparkSession session) {
+ HiveStorageRule rule = new HiveStorageRule(session);
+ rule.setForceDisableSoftAffinity(true);
+ return rule;
+ }
+ });
return BoxedUnit.UNIT;
}
}).enableHiveSupport().config(sparkConf)
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/MppOnTheFlyImpl.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/MppOnTheFlyImpl.java
new file mode 100644
index 0000000000..b942740b14
--- /dev/null
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/MppOnTheFlyImpl.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.engine.spark.job;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.engine.spark.job.stage.BuildParam;
+import org.apache.kylin.engine.spark.job.stage.build.MaterializedFactTableView;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+import org.apache.kylin.metadata.cube.model.LayoutEntity;
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.query.plugin.runtime.MppOnTheFlyProvider;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+import org.apache.kylin.cache.kylin.KylinCacheFileSystem;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+
+@Slf4j
+public class MppOnTheFlyImpl implements MppOnTheFlyProvider {
+
+ /**
+ * Compute a layout on-the-fly for segments that don't have it.
+ * <p/>
+ * Return null if all segments have tne required layout.
+ */
+ @Override
+ public LogicalPlan computeMissingLayout(List<NDataSegment> prunedSegments, long layoutId, SparkSession ss) {
+ List<NDataSegment> missingLayoutSegs = prunedSegments.stream()
+ .filter(seg -> !seg.getLayoutsMap().containsKey(layoutId))
+ .collect(Collectors.toList());
+
+ // quick return if all segments have the layout
+ if (missingLayoutSegs.isEmpty())
+ return null;
+
+ // make a virtual segment, with a merged SegmentRange
+ NDataSegment virtualSeg = virtualSegment(missingLayoutSegs);
+
+ // enable use of local cache of remote source files
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(virtualSeg.getDataflow().getLastModified());
+
+ // compute
+ return computeLayout(virtualSeg, layoutId, ss);
+ }
+
+ private NDataSegment virtualSegment(List<NDataSegment> missingLayoutSegs) {
+ Preconditions.checkState(missingLayoutSegs.size() > 0);
+
+ SegmentRange<?> merged = null;
+ for (NDataSegment seg : missingLayoutSegs) {
+ if (merged == null)
+ merged = seg.getSegRange();
+ else
+ merged = merged.coverWith(seg.getSegRange());
+ }
+
+ NDataSegment ret = new NDataSegment(missingLayoutSegs.get(0).getDataflow(), merged);
+ ret.setId(""); // to skip DFBuilderHelper.checkPointSegment()
+ return ret;
+ }
+
+ /**
+ * Mimic the logic of building cuboid layout from SegmentBuildJob.build()
+ * <p/>
+ * Work hard to reuse the existing SegmentJob & FlatTableAndDictBase etc.
+ */
+ public LogicalPlan computeLayout(NDataSegment virtualSeg, long layoutId, SparkSession ss) {
+ KylinConfig config = virtualSeg.getDataflow().getConfig();
+ KylinBuildEnv.getOrCreate(config);
+ try {
+
+ LayoutEntity layoutEntity = virtualSeg.getIndexPlan().getLayoutEntity(layoutId);
+ MockJobContext jobContext = new MockJobContext(virtualSeg, layoutId, ss);
+ BuildParam buildParam = new BuildParam();
+
+ MaterializedFactTableView tool = new MaterializedFactTableView(jobContext, virtualSeg, buildParam);
+ Dataset<Row> layoutDS = tool.computeLayoutFromSourceAllInOne(layoutEntity);
+
+ return layoutDS == null ? null : layoutDS.queryExecution().logical();
+
+ } finally {
+ KylinBuildEnv.clean();
+ }
+ }
+
+ // just work as a JobContext
+ public static class MockJobContext extends SegmentJob {
+
+ private final NDataSegment seg;
+
+ private MockJobContext(NDataSegment seg, long layoutId, SparkSession ss) {
+ this.seg = seg;
+ this.ss = ss;
+
+ this.dataflowId = seg.getDataflow().getId();
+ this.indexPlan = seg.getIndexPlan();
+ this.readOnlySegments = ImmutableSet.of(seg);
+ this.readOnlyLayouts = ImmutableSet.of(seg.getIndexPlan().getLayoutEntity(layoutId));
+ }
+
+ @Override
+ public KylinConfig getConfig() {
+ return seg.getDataflow().getConfig();
+ }
+
+ @Override
+ public SparkSession getSparkSession() {
+ return ss;
+ }
+
+ @Override
+ public boolean isPartialBuild() {
+ return false;
+ }
+
+ // empty
+ @Override
+ public IJobProgressReport getReport() {
+ return new IJobProgressReport() {
+ @Override
+ public boolean updateSparkJobInfo(Map<String, String> params, String url, String json) {
+ return false;
+ }
+ @Override
+ public boolean updateSparkJobExtraInfo(Map<String, String> params, String url, String project, String jobId, Map<String, String> extraInfo) {
+ return false;
+ }
+ };
+ }
+
+ // empty
+ @Override
+ protected void doExecute() throws Exception {
+ // nothing to do
+ }
+ }
+}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala
index 9076afb6a9..1ae54f626c 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFBuilderHelper.scala
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.spark.builder
+import org.apache.commons.lang3.StringUtils
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.persistence.transaction.UnitOfWork
import org.apache.kylin.engine.spark.job.NSparkCubingUtil
@@ -84,6 +85,11 @@ object DFBuilderHelper extends Logging {
}
def checkPointSegment(readOnlySeg: NDataSegment, checkpointOps: NDataSegment => Unit): NDataSegment = {
+ // skip invalid or virtual segment
+ if (StringUtils.isEmpty(readOnlySeg.getId)) {
+ return readOnlySeg
+ }
+
// read basic infos from the origin segment
val segId = readOnlySeg.getId
val dfId = readOnlySeg.getDataflow.getId
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FiltersUtil.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FiltersUtil.scala
index b4fbfd4907..e7862c2724 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FiltersUtil.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FiltersUtil.scala
@@ -87,7 +87,7 @@ object FiltersUtil extends Logging {
val segRange = flatTable.getSegmentRange
if (segRange != null && !segRange.isInfinite) {
var afterConvertPartition = partDesc.getPartitionConditionBuilder
- .buildDateRangeCondition(partDesc, null, segRange)
+ .buildDateRangeCondition(partDesc, flatTable.getDataSegment, segRange)
afterConvertPartition = CreateFlatTable.replaceDot(afterConvertPartition, model)
logInfo(s"Partition filter $afterConvertPartition")
afterFilter = afterFilter.where(afterConvertPartition)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
index 3ba93d705e..1c88b4e600 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
@@ -37,7 +37,7 @@ object FlatTableHelper extends Logging {
val segRange = flatTable.getSegRange
if (segRange != null && !segRange.isInfinite) {
var afterConvertPartition = partDesc.getPartitionConditionBuilder
- .buildDateRangeCondition(partDesc, null, segRange)
+ .buildDateRangeCondition(partDesc, flatTable.getSegment, segRange)
if (needReplaceDot) afterConvertPartition = replaceDot(afterConvertPartition, model)
logInfo(s"Partition filter $afterConvertPartition")
afterFilter = afterFilter.where(afterConvertPartition)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
index 05e0f55327..70c0208e43 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentBuildJob.java
@@ -42,6 +42,7 @@ import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.spark.sql.hive.utils.ResourceDetectUtils;
import org.apache.spark.tracker.BuildContext;
+import org.apache.kylin.fileseg.FileSegments;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -95,9 +96,9 @@ public class SegmentBuildJob extends SegmentJob {
@Override
protected final void doExecute() throws Exception {
- log.info("Start sub stage {}" + REFRESH_SNAPSHOTS.name());
+ log.info("Start sub stage {}", REFRESH_SNAPSHOTS.name());
REFRESH_SNAPSHOTS.create(this, null, null).toWork();
- log.info("End sub stage {}" + REFRESH_SNAPSHOTS.name());
+ log.info("End sub stage {}", REFRESH_SNAPSHOTS.name());
buildContext = new BuildContext(getSparkSession().sparkContext(), config);
buildContext.appStatusTracker().startMonitorBuildResourceState();
@@ -160,6 +161,8 @@ public class SegmentBuildJob extends SegmentJob {
} catch (IOException e) {
Throwables.propagate(e);
+ } finally {
+ FileSegments.clearFileSegFilterLocally();
}
});
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java
index bdbfcfaafd..5c0df3b2a8 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/SegmentJob.java
@@ -64,7 +64,7 @@ public abstract class SegmentJob extends SparkApplication {
protected String dataflowId;
// In order to support the cost based planner, `readOnlyLayouts` can be changed
- private Set<LayoutEntity> readOnlyLayouts;
+ protected Set<LayoutEntity> readOnlyLayouts;
protected Set<NDataSegment> readOnlySegments;
// Resource detection results output path
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
index d3f863a313..71d6f109ff 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/BuildStage.scala
@@ -408,7 +408,7 @@ abstract class BuildStage(private val jobContext: SegmentJob,
(id, columns)
}.toMap
- private def columnsFromFlatTable(index: IndexEntity): Seq[String] = {
+ def columnsFromFlatTable(index: IndexEntity): Seq[String] = {
val columns = mutable.Set[String]()
// Dimension columns
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index 9cd5487904..4634b77096 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -306,7 +306,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
}
val condition = descDRP.getPartitionConditionBuilder //
- .buildDateRangeCondition(descDRP, null, segmentRange)
+ .buildDateRangeCondition(descDRP, tableDesc.getDataSegment, segmentRange)
logInfo(s"Apply PARTITION-CONDITION $condition segment $segmentId")
originDS.where(condition)
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
index 19b5cd073c..35232cdeb7 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/MaterializedFactTableView.scala
@@ -20,7 +20,12 @@ package org.apache.kylin.engine.spark.job.stage.build
import org.apache.kylin.engine.spark.job.SegmentJob
import org.apache.kylin.engine.spark.job.stage.BuildParam
-import org.apache.kylin.metadata.cube.model.NDataSegment
+import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet
+import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree
+import org.apache.kylin.metadata.cube.cuboid.AdaptiveSpanningTree.AdaptiveTreeBuilder
+import org.apache.kylin.metadata.cube.model.{LayoutEntity, NDataSegment}
+import org.apache.spark.sql.{Dataset, Row}
class MaterializedFactTableView(jobContext: SegmentJob, dataSegment: NDataSegment, buildParam: BuildParam)
extends FlatTableAndDictBase(jobContext, dataSegment, buildParam) {
@@ -34,4 +39,29 @@ class MaterializedFactTableView(jobContext: SegmentJob, dataSegment: NDataSegmen
}
override def getStageName: String = "MaterializedFactTableView"
+
+ // just moved from 4.5.x, needs refactor...
+ def computeLayoutFromSourceAllInOne(layoutEntity: LayoutEntity): Dataset[Row] = {
+ val spanTree = new AdaptiveSpanningTree(config,
+ new AdaptiveTreeBuilder(dataSegment, ImmutableSet.of(layoutEntity)))
+ buildParam.setSpanningTree(spanTree)
+
+ val flatTableDesc = new SegmentFlatTableDesc(config, dataSegment, spanTree)
+ buildParam.setFlatTableDesc(flatTableDesc)
+
+ val factTableDS = newFastFactTableDS()
+ buildParam.setFactTableDS(factTableDS)
+
+ val dict = buildDictIfNeed()
+ buildParam.setDict(dict)
+
+ val flatTable = dict // YES, the 'dict' is actually 'flatTable'
+ buildParam.setFlatTable(flatTable)
+
+ // flat table ==> layout DS
+ val parentDS = flatTable.select(columnsFromFlatTable(layoutEntity.getIndex).map(org.apache.spark.sql.functions.col): _*)
+ val layoutDS = wrapLayoutDS(layoutEntity, parentDS)
+
+ layoutDS
+ }
}
diff --git a/src/spark-project/kylin-soft-affinity-cache-hadoop3/pom.xml b/src/spark-project/kylin-soft-affinity-cache-hadoop3/pom.xml
new file mode 100644
index 0000000000..1e3ae7a623
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache-hadoop3/pom.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>kylin-soft-affinity-cache-hadoop3</artifactId>
+ <packaging>jar</packaging>
+ <url>http://kyligence.io</url>
+ <description>Kylin Soft Affinity Cache with Hadoop3</description>
+ <name>Kylin - Soft Affinity Local Cache(Hadoop3)</name>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <sonar.sources>src/main/java,src/main/scala</sonar.sources>
+ </properties>
+
+ <parent>
+ <artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
+ <version>5.0.0-SNAPSHOT</version>
+ <relativePath>../../../pom.xml</relativePath>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-soft-affinity-cache</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.12</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.12</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/src/spark-project/kylin-soft-affinity-cache-hadoop3/src/main/java/io/kyligence/kap/cache/kylin/KylinCacheFileSystemHadoop3.java b/src/spark-project/kylin-soft-affinity-cache-hadoop3/src/main/java/io/kyligence/kap/cache/kylin/KylinCacheFileSystemHadoop3.java
new file mode 100644
index 0000000000..d61b1b5b3c
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache-hadoop3/src/main/java/io/kyligence/kap/cache/kylin/KylinCacheFileSystemHadoop3.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.kyligence.kap.cache.kylin;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class KylinCacheFileSystemHadoop3 extends KylinCacheFileSystem {
+
+ @Override
+ public FutureDataInputStreamBuilder openFile(final Path path) throws UnsupportedOperationException {
+ return new FutureDataInputStreamBuilderImpl(fs, path) {
+ @Override
+ public CompletableFuture<FSDataInputStream> build() throws IllegalArgumentException, UnsupportedOperationException, IOException {
+ return CompletableFuture.completedFuture(KylinCacheFileSystemHadoop3.this.open(path));
+ }
+ };
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/pom.xml b/src/spark-project/kylin-soft-affinity-cache/pom.xml
index 97a5f2cd04..cfa824c8ef 100644
--- a/src/spark-project/kylin-soft-affinity-cache/pom.xml
+++ b/src/spark-project/kylin-soft-affinity-cache/pom.xml
@@ -40,6 +40,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metadata</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
@@ -86,6 +90,23 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <classifier>core</classifier>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
@@ -131,6 +152,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- Spark test-->
<dependency>
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java
index e9740e4f4e..8fee3aee3b 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AbstractCacheFileSystem.java
@@ -20,31 +20,31 @@ package org.apache.kylin.cache.fs;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.sparkproject.guava.hash.Hashing.sha256;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.commons.lang3.time.StopWatch;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.kylin.cache.utils.ReflectionUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
-import org.apache.kylin.guava30.shaded.common.cache.CacheLoader;
-import org.apache.kylin.guava30.shaded.common.cache.LoadingCache;
+import org.apache.kylin.common.QueryContext;
import alluxio.client.file.CacheContext;
import alluxio.client.file.URIStatus;
@@ -57,11 +57,15 @@ import alluxio.hadoop.HadoopUtils;
import alluxio.metrics.MetricsConfig;
import alluxio.metrics.MetricsSystem;
import alluxio.wire.FileInfo;
+import org.apache.kylin.cache.utils.Hadoop3CompaUtil;
+import org.apache.kylin.fileseg.FileSegments;
+import org.apache.kylin.fileseg.FileSegments.SourceFilePredicate;
+import org.apache.kylin.fileseg.FileSegmentsDetector;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public abstract class AbstractCacheFileSystem extends FilterFileSystem {
- private static final Logger logger = LoggerFactory.getLogger(AbstractCacheFileSystem.class);
-
protected URI uri;
protected String originalScheme;
protected int bufferSize = 4096;
@@ -71,21 +75,20 @@ public abstract class AbstractCacheFileSystem extends FilterFileSystem {
protected boolean useBufferFileInputStream = false;
protected HadoopFileOpener mHadoopFileOpener;
protected LocalCacheFileInStream.FileInStreamOpener mAlluxioFileOpener;
- protected CacheManager mCacheManager;
protected AlluxioConfiguration mAlluxioConf;
-
- protected LoadingCache<Path, FileStatus> fileStatusCache;
+ protected ManagerOfCacheFileContent mCacheManager;
+ protected ManagerOfCacheFileStatus mStatusCache;
+ protected Map<Path, Long> lastFetchTimeOfPaths = new ConcurrentHashMap<>();
protected static final Map<String, String> schemeClassMap = Stream
.of(new AbstractMap.SimpleImmutableEntry<>("file", "org.apache.hadoop.fs.LocalFileSystem"),
new AbstractMap.SimpleImmutableEntry<>("viewfs", "org.apache.hadoop.fs.viewfs.ViewFileSystem"),
new AbstractMap.SimpleImmutableEntry<>("s3a", "org.apache.hadoop.fs.s3a.S3AFileSystem"),
- new AbstractMap.SimpleImmutableEntry<>("s3", "org.apache.hadoop.fs.s3.S3FileSystem"),
+ new AbstractMap.SimpleImmutableEntry<>("s3", "org.apache.hadoop.fs.s3a.S3AFileSystem"),
new AbstractMap.SimpleImmutableEntry<>("s3n", "org.apache.hadoop.fs.s3native.NativeS3FileSystem"),
new AbstractMap.SimpleImmutableEntry<>("hdfs", "org.apache.hadoop.hdfs.DistributedFileSystem"),
new AbstractMap.SimpleImmutableEntry<>("wasb", "org.apache.hadoop.fs.azure.NativeAzureFileSystem"),
- new AbstractMap.SimpleImmutableEntry<>("wasbs",
- "org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure"),
+ new AbstractMap.SimpleImmutableEntry<>("wasbs", "org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure"),
new AbstractMap.SimpleImmutableEntry<>("alluxio", "alluxio.hadoop.FileSystem"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
@@ -98,11 +101,12 @@ public abstract class AbstractCacheFileSystem extends FilterFileSystem {
}
FileSystem fs;
try {
+ @SuppressWarnings("unchecked")
Class<? extends FileSystem> clazz = (Class<? extends FileSystem>) conf
.getClassByName(schemeClassMap.get(uri.getScheme()));
fs = ReflectionUtils.newInstance(clazz, conf);
fs.initialize(uri, conf);
- logger.info("Create filesystem {} for scheme {} .", schemeClassMap.get(uri.getScheme()), uri.getScheme());
+ log.info("Create filesystem {} for scheme {}", schemeClassMap.get(uri.getScheme()), uri.getScheme());
} catch (ClassNotFoundException e) {
throw new IOException("Can not found FileSystem Clazz for scheme: " + uri.getScheme());
}
@@ -114,16 +118,14 @@ public abstract class AbstractCacheFileSystem extends FilterFileSystem {
mAlluxioFileOpener = status -> new AlluxioHdfsInputStream(mHadoopFileOpener.open(status));
mAlluxioConf = HadoopUtils.toAlluxioConf(conf);
+
// Handle metrics
Properties metricsProperties = new Properties();
for (Map.Entry<String, String> entry : conf) {
metricsProperties.setProperty(entry.getKey(), entry.getValue());
}
MetricsSystem.startSinksFromConfig(new MetricsConfig(metricsProperties));
- mCacheManager = CacheManager.Factory.get(mAlluxioConf);
- if (mCacheManager == null) {
- throw new IOException("CacheManager is null !");
- }
+ mCacheManager = new ManagerOfCacheFileContent(CacheManager.Factory.get(mAlluxioConf));
}
@Override
@@ -133,13 +135,13 @@ public abstract class AbstractCacheFileSystem extends FilterFileSystem {
this.fs = createInternalFS(name, conf);
this.statistics = (FileSystem.Statistics) ReflectionUtil.getFieldValue(this.fs, "statistics");
if (null == this.statistics) {
- logger.info("======= original statistics is null.");
+ log.info("======= original statistics is null.");
} else {
- logger.info("======= original statistics is {} {}.", this.statistics.getScheme(), this.statistics);
+ log.info("======= original statistics is {} {}.", this.statistics.getScheme(), this.statistics);
}
super.initialize(name, conf);
this.setConf(conf);
- logger.info("======= current statistics is {} {}.", this.statistics.getScheme(), this.statistics);
+ log.info("======= current statistics is {} {}.", this.statistics.getScheme(), this.statistics);
this.bufferSize = conf.getInt(CacheFileSystemConstants.PARAMS_KEY_IO_FILE_BUFFER_SIZE,
CacheFileSystemConstants.PARAMS_KEY_IO_FILE_BUFFER_SIZE_DEFAULT_VALUE);
@@ -157,49 +159,53 @@ public abstract class AbstractCacheFileSystem extends FilterFileSystem {
CacheFileSystemConstants.PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM_DEFAULT_VALUE);
// create FileStatus cache
- long fileStatusTTL = conf.getLong(CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_TTL,
- CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_TTL_DEFAULT_VALUE);
- long fileStatusMaxSize = conf.getLong(CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE,
- CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE_DEFAULT_VALUE);
- CacheLoader<Path, FileStatus> fileStatusCacheLoader = new CacheLoader<Path, FileStatus>() {
- @Override
- public FileStatus load(Path path) throws Exception {
- return getFileStatusForCache(path);
- }
- };
- this.fileStatusCache = CacheBuilder.newBuilder().maximumSize(fileStatusMaxSize)
- .expireAfterAccess(fileStatusTTL, TimeUnit.SECONDS).recordStats().build(fileStatusCacheLoader);
+ if (this.useFileStatusCache) {
+ long fileStatusTTL = conf.getLong(CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_TTL,
+ CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_TTL_DEFAULT_VALUE);
+ long fileStatusMaxSize = conf.getLong(CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE,
+ CacheFileSystemConstants.PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE_DEFAULT_VALUE);
+ this.mStatusCache = new ManagerOfCacheFileStatus(fileStatusTTL, fileStatusMaxSize,
+ p -> {
+ try {
+ return fs.getFileStatus(p);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ },
+ p -> {
+ try {
+ List<LocatedFileStatus> ret = Hadoop3CompaUtil.RemoteIterators.toList(fs.listLocatedStatus(p));
+ return ret.toArray(new LocatedFileStatus[0]);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ );
+ }
// create LocalCacheFileSystem if it needs
- if (this.isUseLocalCache()) {
+ if (this.useLocalCache) {
this.createLocalCacheManager(conf);
- logger.info("Create LocalCacheFileSystem successfully .");
+ log.info("Create LocalCacheFileSystem successfully .");
}
}
- protected FileStatus getFileStatusForCache(Path path) throws IOException {
- return this.fs.getFileStatus(path);
- }
-
@Override
public String getScheme() {
return this.originalScheme;
}
- @Override
- public FSDataInputStream open(Path f) throws IOException {
- return open(f, bufferSize);
- }
-
/**
* Check whether it needs to cache data on the current executor
*/
- public abstract boolean isUseLocalCacheForTargetExecs();
+ protected abstract boolean isUseLocalCacheForCurrentExecutor();
+
+ protected abstract long getAcceptCacheTime();
/**
* Wrap FileStatus to Alluxio FileInfo
*/
- public FileInfo wrapFileInfo(FileStatus fileStatus) {
+ private FileInfo wrapFileInfo(FileStatus fileStatus) {
return (new FileInfo().setLength(fileStatus.getLen()).setPath(fileStatus.getPath().toString())
.setFolder(fileStatus.isDirectory()).setBlockSizeBytes(fileStatus.getBlockSize())
.setLastModificationTimeMs(fileStatus.getModificationTime())
@@ -211,71 +217,158 @@ public abstract class AbstractCacheFileSystem extends FilterFileSystem {
if (size < this.bufferSize) {
size = this.bufferSize;
}
-
return size;
}
+ @Override
+ public FSDataInputStream open(Path f) throws IOException {
+ return open(f, bufferSize);
+ }
+
@Override
public FSDataInputStream open(Path p, int bufferSize) throws IOException {
- return this.open(p, bufferSize, this.isUseLocalCacheForTargetExecs());
+ return this.open(p, bufferSize, this.isUseLocalCacheForCurrentExecutor());
}
public FSDataInputStream open(Path p, int bufferSize, boolean useLocalCacheForExec) throws IOException {
+ // quick return if cache is not enabled
+ boolean enabled = (this.useLocalCache && this.mCacheManager != null && useLocalCacheForExec);
+ if (!enabled) {
+ log.debug("Use original FileSystem to open file {} .", p);
+ return fs.open(p, bufferSize);
+ }
+
Path f = this.fs.makeQualified(p);
+ FileStatus fileStatus = this.getFileStatus(f);
+ FileInfo fileInfo = wrapFileInfo(fileStatus);
+
+ // FilePath is a unique identifier for a file, however it can be a long string
+ // hence using md5 hash of the file path as the identifier in the cache.
+ String fileHashFrom = "" + fileStatus.getPath() + fileStatus.getLen() + fileStatus.getModificationTime();
+ String fileId = sha256().hashString(fileHashFrom, UTF_8).toString();
+ CacheContext context = CacheContext.defaults().setCacheIdentifier(fileId);
+ URIStatus status = new URIStatus(fileInfo, context);
+ int cachedCount = mCacheManager.countCachedPages(fileId);
+
+ if (this.useLegacyFileInputStream) {
+ log.debug("Local cache (Legacy) opens file ({} pages cached) {} .", cachedCount, f);
+ return new FSDataInputStream(new AlluxioHdfsFileInputStream(
+ new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf),
+ statistics));
+ } else if (this.useBufferFileInputStream) {
+ log.debug("Local cache (Direct-Buffer) opens file ({} pages cached) {} .", cachedCount, f);
+ return new FSDataInputStream(new CacheFileInputStream(f,
+ new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf), null,
+ statistics, checkBufferSize(bufferSize)));
+ } else {
+ throw new IllegalStateException();
+ }
+ }
- if (this.isUseLocalCache() && this.mCacheManager != null && useLocalCacheForExec) {
- FileStatus fileStatus = this.getFileStatus(f);
- FileInfo fileInfo = wrapFileInfo(fileStatus);
- // FilePath is a unique identifier for a file, however it can be a long string
- // hence using md5 hash of the file path as the identifier in the cache.
- CacheContext context = CacheContext.defaults()
- .setCacheIdentifier(sha256().hashString(fileStatus.getPath().toString(), UTF_8).toString());
- URIStatus status = new URIStatus(fileInfo, context);
-
- if (this.useLegacyFileInputStream) {
- logger.info("Use local cache Legacy FileSystem to open file {} .", f);
- return new FSDataInputStream(new AlluxioHdfsFileInputStream(
- new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf),
- statistics));
- } else if (this.useBufferFileInputStream) {
- logger.info("Use local cache Buffer FileSystem to open file {} .", f);
- return new FSDataInputStream(new CacheFileInputStream(f,
- new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf), null,
- statistics, checkBufferSize(bufferSize)));
+ @Override
+ public FileStatus getFileStatus(Path p) throws IOException {
+ if (!this.useFileStatusCache) {
+ StopWatch w = StopWatch.createStarted();
+ try {
+ return fs.getFileStatus(p);
+ } finally {
+ log.warn("Slow fs operation took {} ms: {}({})", w.getTime(), "getFileStatus", p);
}
}
- logger.info("Use original FileSystem to open file {} .", f);
- return super.open(f, bufferSize);
+
+ p = this.fs.makeQualified(p);
+ this.statistics.incrementReadOps(1);
+
+ mStatusCache.ensureCacheCreateTime(p, getAcceptCacheTime());
+ return mStatusCache.getFileStatus(p);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path p) throws IOException {
+ return _listLocatedStatus(p);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path p) throws IOException {
+ return Hadoop3CompaUtil.RemoteIterators.remoteIteratorFromIterator(filteredStatusStream(p).iterator());
}
@Override
- public FileStatus getFileStatus(Path f) throws IOException {
- if (isUseFileStatusCache()) {
+ public RemoteIterator<FileStatus> listStatusIterator(Path p) throws IOException {
+ return Hadoop3CompaUtil.RemoteIterators.remoteIteratorFromIterator(
+ filteredStatusStream(p)
+ .map(FileStatus.class::cast)
+ .iterator());
+ }
+
+ private Stream<LocatedFileStatus> filteredStatusStream(Path p) throws IOException {
+ LocatedFileStatus[] ret = _listLocatedStatus(p);
+
+ Optional<SourceFilePredicate> filter = FileSegments.getFileSegFilterLocally(p.toString());
+ if (filter.isPresent()) {
+ return Arrays.stream(ret).filter(f -> filter.get().checkFile(f));
+ } else {
+ return Arrays.stream(ret);
+ }
+
+ }
+
+ private LocatedFileStatus[] _listLocatedStatus(Path p) throws IOException {
+ LocatedFileStatus[] result;
+
+ if (this.useFileStatusCache) {
+ p = this.fs.makeQualified(p);
this.statistics.incrementReadOps(1);
- long start = System.currentTimeMillis();
- FileStatus fileStatus;
- Path p = this.fs.makeQualified(f);
+
+ long requiredCacheTime = getAcceptCacheTime();
+
+ // path is going to be fetched by the required cached time, use this info to update QueryContext.dataFetchTime
+ updateDataFetchTimeInContext(p, requiredCacheTime);
+
+ mStatusCache.ensureCacheCreateTime(p, requiredCacheTime);
+ result = mStatusCache.listChildren(p);
+
+ } else {
+ StopWatch w = StopWatch.createStarted();
try {
- fileStatus = this.fileStatusCache.get(p);
- } catch (ExecutionException e) {
- if (e.getCause() instanceof FileNotFoundException) {
- logger.info("Get file status from cache error: {}", p);
- throw new FileNotFoundException("File does not exist: " + p);
- }
- return null;
+ List<LocatedFileStatus> list = Hadoop3CompaUtil.RemoteIterators.toList(fs.listLocatedStatus(p));
+ result = list.toArray(new LocatedFileStatus[0]);
+ } finally {
+ log.warn("Slow fs operation took {} ms: {}({})", w.getTime(), "listLocatedStatus", p);
}
- logger.info("Get file status {} from cache took: {}", f, (System.currentTimeMillis() - start));
- return fileStatus;
- } else {
- return super.getFileStatus(f);
}
+
+ FileSegmentsDetector.onObserveFileSystemListFileStatus(p, result);
+
+ return result;
+ }
+
+ private void updateDataFetchTimeInContext(Path p, long requiredCacheTime) {
+ QueryContext context = QueryContext.current();
+ if (context == null)
+ return; // not in KE process, thus has no query context
+
+ long lastFetchTime = lastFetchTimeOfPaths.getOrDefault(p, 0L);
+ lastFetchTime = Long.max(lastFetchTime, requiredCacheTime);
+
+ if (lastFetchTime == 0) { // accessing the path for the very first time?
+ lastFetchTime = System.currentTimeMillis();
+ }
+
+ // path is going to be fetched by the required cached time, use this info to update QueryContext.dataFetchTime
+ context.getMetrics().addDataFetchTime(lastFetchTime);
+ lastFetchTimeOfPaths.put(p, lastFetchTime);
+ }
+
+ public long countCachedFilePages() {
+ return mCacheManager.countTotalCachedPages();
}
- public boolean isUseLocalCache() {
- return useLocalCache;
+ public long countCachedFileStatus() {
+ return mStatusCache.size();
}
- public boolean isUseFileStatusCache() {
- return useFileStatusCache;
+ public long countCachedFileStatusEvictions() {
+ return mStatusCache.countEvictions();
}
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java
index e2948e7d04..dd07994f6c 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/AlluxioHdfsFileInputStream.java
@@ -86,7 +86,7 @@ public class AlluxioHdfsFileInputStream extends InputStream
int data = fileInStream.read();
if (data != -1 && statistics != null) {
statistics.incrementBytesRead(1);
- logger.debug("Read one byte.");
+ logger.trace("Read one byte.");
}
return data;
}
@@ -105,7 +105,7 @@ public class AlluxioHdfsFileInputStream extends InputStream
int bytesRead = fileInStream.read(buffer, offset, length);
if (bytesRead != -1 && statistics != null) {
statistics.incrementBytesRead(bytesRead);
- logger.debug("Read {} bytes.", bytesRead);
+ logger.trace("Read {} bytes.", bytesRead);
}
return bytesRead;
}
@@ -136,7 +136,7 @@ public class AlluxioHdfsFileInputStream extends InputStream
}
if (bytesRead != -1 && statistics != null) {
statistics.incrementBytesRead(bytesRead);
- logger.debug("Read {} byte buffer {}.", bytesRead, buf.hasArray());
+ logger.trace("Read {} byte buffer {}.", bytesRead, buf.hasArray());
}
return bytesRead;
}
@@ -150,7 +150,7 @@ public class AlluxioHdfsFileInputStream extends InputStream
int bytesRead = fileInStream.positionedRead(position, buffer, offset, length);
if (bytesRead != -1 && statistics != null) {
statistics.incrementBytesRead(bytesRead);
- logger.debug("Read {} {} byte buffer.", position, bytesRead);
+ logger.trace("Read {} {} byte buffer.", position, bytesRead);
}
return bytesRead;
}
@@ -170,7 +170,7 @@ public class AlluxioHdfsFileInputStream extends InputStream
}
totalBytesRead += bytesRead;
}
- logger.info("Read fully {} {} byte buffer.", position, totalBytesRead);
+ logger.trace("Read fully {} {} byte buffer.", position, totalBytesRead);
}
@Override
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java
index 5a34b127bc..d1f89fb8b6 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/CacheFileSystemConstants.java
@@ -22,40 +22,28 @@ public class CacheFileSystemConstants {
private CacheFileSystemConstants() {
}
- public static final String PARAMS_KEY_USE_CACHE =
- "spark.kylin.local-cache.enabled";
-
+ public static final String PARAMS_KEY_USE_CACHE = "spark.kylin.local-cache.enabled";
public static final boolean PARAMS_KEY_USE_CACHE_DEFAULT_VALUE = false;
- public static final String PARAMS_KEY_USE_FILE_STATUS_CACHE =
- "spark.kylin.file-status-cache.enabled";
-
+ public static final String PARAMS_KEY_USE_FILE_STATUS_CACHE = "spark.kylin.file-status-cache.enabled";
public static final boolean PARAMS_KEY_USE_FILE_STATUS_CACHE_DEFAULT_VALUE = false;
public static final String PARAMS_KEY_IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
-
public static final int PARAMS_KEY_IO_FILE_BUFFER_SIZE_DEFAULT_VALUE = 65536;
- public static final String PARAMS_KEY_FILE_STATUS_CACHE_TTL =
- "spark.kylin.local-cache.filestatus-cache-ttl";
-
- public static final long PARAMS_KEY_FILE_STATUS_CACHE_TTL_DEFAULT_VALUE = 3600L;
-
- public static final String PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE =
- "spark.kylin.local-cache.filestatus-cache-max-size";
+ public static final String PARAMS_KEY_FILE_STATUS_CACHE_TTL = "spark.kylin.local-cache.filestatus-cache-ttl";
+ public static final long PARAMS_KEY_FILE_STATUS_CACHE_TTL_DEFAULT_VALUE = 36000L;
+ public static final String PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE = "spark.kylin.local-cache.filestatus-cache-max-size";
public static final long PARAMS_KEY_FILE_STATUS_CACHE_MAX_SIZE_DEFAULT_VALUE = 10000L;
- public static final String PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM =
- "spark.kylin.local-cache.use.legacy.file-input-stream";
-
+ public static final String PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM = "spark.kylin.local-cache.use.legacy.file-input-stream";
public static final boolean PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM_DEFAULT_VALUE = false;
- public static final String PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM =
- "spark.kylin.local-cache.use.buffer.file-input-stream";
-
+ public static final String PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM = "spark.kylin.local-cache.use.buffer.file-input-stream";
public static final boolean PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM_DEFAULT_VALUE = false;
- public static final String PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES =
- "spark.kylin.local-cache.for.current.files";
+ public static final String PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES = "spark.kylin.local-cache.for.current.files";
+
+ public static final String PARAMS_KEY_ACCEPT_CACHE_TIME = "spark.kylin.local-cache.accept-cache-time";
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContent.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContent.java
new file mode 100644
index 0000000000..7bf123f202
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContent.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.cache.fs;
+
+import alluxio.client.file.CacheContext;
+import alluxio.client.file.cache.CacheManager;
+import alluxio.client.file.cache.PageId;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.BitSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A wrapper around alluxio CacheManager, enriching some file operations, like
+ * 1) tracking cache time at file level
+ * 2) deleting cache at file level, etc.
+ */
+@RequiredArgsConstructor
+@Slf4j
+public class ManagerOfCacheFileContent implements CacheManager {
+
+ private final CacheManager under;
+ private final ConcurrentHashMap<String, FileMeta> fileMetas = new ConcurrentHashMap<>();
+
+ @Override
+ public boolean put(PageId pageId, byte[] page, CacheContext cacheContext) {
+ boolean ok = under.put(pageId, page, cacheContext);
+ markFilePageCached(pageId);
+ return ok;
+ }
+
+ @Override
+ public int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, int offsetInBuffer, CacheContext cacheContext) {
+ int bytesRead = under.get(pageId, pageOffset, bytesToRead, buffer, offsetInBuffer, cacheContext);
+ if (bytesRead > 0)
+ markFilePageCached(pageId);
+ return bytesRead;
+ }
+
+ @Override
+ public boolean delete(PageId pageId) {
+ boolean ok = under.delete(pageId);
+ markFilePageRemoved(pageId);
+ return ok;
+ }
+
+ @Override
+ public State state() {
+ return under.state();
+ }
+
+ @Override
+ public void close() throws Exception {
+ under.close();
+ }
+
+ @RequiredArgsConstructor
+ static class FileMeta {
+ final String fileId;
+ final long cacheCreateTime = System.currentTimeMillis();
+ final BitSet cachedPageIndex = new BitSet();
+ }
+
+ public int countCachedPages(String fileId) {
+ FileMeta meta = fileMetas.get(fileId);
+ if (meta == null)
+ return 0;
+ else
+ return meta.cachedPageIndex.cardinality();
+ }
+
+ public long countTotalCachedPages() {
+ long sum = 0;
+ for (FileMeta meta : fileMetas.values()) {
+ sum += meta.cachedPageIndex.cardinality();
+ }
+ return sum;
+ }
+
+ protected void markFilePageCached(PageId pageId) {
+ fileMetas.compute(pageId.getFileId(),
+ (fileId, fmeta) -> {
+ if (fmeta == null)
+ fmeta = new FileMeta(fileId);
+ fmeta.cachedPageIndex.set((int) pageId.getPageIndex());
+ return fmeta;
+ });
+ }
+
+ protected void markFilePageRemoved(PageId pageId) {
+ fileMetas.compute(pageId.getFileId(),
+ (fileId, fmeta) -> {
+ if (fmeta == null)
+ fmeta = new FileMeta(fileId);
+ fmeta.cachedPageIndex.clear((int) pageId.getPageIndex());
+ return fmeta;
+ });
+ }
+
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatus.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatus.java
new file mode 100644
index 0000000000..69ee033481
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatus.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.cache.fs;
+
+import org.apache.kylin.cache.kylin.KylinCacheFileSystem;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
+import org.apache.kylin.guava30.shaded.common.cache.CacheLoader;
+import org.apache.kylin.guava30.shaded.common.cache.LoadingCache;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.util.concurrent.UncheckedExecutionException;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+/**
+ * A cache of file status.
+ */
+@Slf4j
+public class ManagerOfCacheFileStatus {
+
+ private final LoadingCache<Path, FileStatusEntry> statusCache;
+ private final LoadingCache<Path, FileStatusEntry> childrenCache;
+ private int countOfEvict; // for test
+
+ public ManagerOfCacheFileStatus(long ttlSec, long maxSize,
+ Function<Path, FileStatus> statusLoader,
+ Function<Path, LocatedFileStatus[]> statusListLoader) {
+ this.statusCache = CacheBuilder.newBuilder()
+ .maximumSize(maxSize)
+ .expireAfterAccess(ttlSec, TimeUnit.SECONDS)
+ .recordStats()
+ .build(new CacheLoader<Path, FileStatusEntry>() {
+ @Override
+ public FileStatusEntry load(Path p) {
+ StopWatch w = StopWatch.createStarted();
+ long cacheTime = KylinCacheFileSystem.getAcceptCacheTimeLocally();
+
+ FileStatus status = statusLoader.apply(p);
+
+ log.debug("Slow fs operation took {} ms, now cached at {}: {}({})", w.getTime(), cacheTime, "getFileStatus", p);
+ return new FileStatusEntry(p, status, null, cacheTime);
+ }
+ });
+ this.childrenCache = CacheBuilder.newBuilder()
+ .maximumSize(maxSize)
+ .expireAfterAccess(ttlSec, TimeUnit.SECONDS)
+ .recordStats()
+ .build(new CacheLoader<Path, FileStatusEntry>() {
+ @Override
+ public FileStatusEntry load(Path p) {
+ StopWatch w = StopWatch.createStarted();
+ long cacheTime = KylinCacheFileSystem.getAcceptCacheTimeLocally();
+
+ LocatedFileStatus[] children = statusListLoader.apply(p);
+ for (LocatedFileStatus child : children) {
+ statusCache.put(child.getPath(), new FileStatusEntry(child.getPath(), child, null, cacheTime));
+ }
+
+ log.debug("Slow fs operation took {} ms, now cached at {}: got {} files from {}({})", w.getTime(), cacheTime, children.length, "listFileStatus", p);
+ return new FileStatusEntry(p, null, children, cacheTime);
+ }
+ });
+ }
+
+ @RequiredArgsConstructor
+ private static class FileStatusEntry {
+ final Path p;
+ final FileStatus status;
+ final LocatedFileStatus[] children;
+ final long cacheCreateTime;
+ }
+
+ /**
+ * Ensure the cache create time of a file is later than a required time.
+ * In other word, user wants to see the refresh data after a required time.
+ * Stale cache must be cleared by this method.
+ */
+ public void ensureCacheCreateTime(Path p, long requiredCacheTime) {
+ if (requiredCacheTime <= 0)
+ return;
+
+ List<LoadingCache<Path, FileStatusEntry>> caches = ImmutableList.of(statusCache, childrenCache);
+ for (LoadingCache<Path, FileStatusEntry> c : caches) {
+ FileStatusEntry entry = c.asMap().get(p);
+ if (entry != null && entry.cacheCreateTime < requiredCacheTime) {
+ log.debug("(accept-cache-time:{}) Evict file status cache for {}", requiredCacheTime, p);
+ c.asMap().remove(p);
+ countOfEvict++;
+ }
+ }
+ }
+
+ public FileStatus getFileStatus(Path p) throws IOException {
+ try {
+ return this.statusCache.get(p).status;
+ } catch (Exception e) {
+ throw makeIOException(e);
+ }
+ }
+
+ public LocatedFileStatus[] listChildren(Path p) throws IOException {
+ try {
+ FileStatusEntry entry = this.childrenCache.get(p);
+ log.trace("Return cached at {}: got {} files from {}({})", entry.cacheCreateTime, entry.children.length, "listFileStatus", p);
+ return entry.children;
+ } catch (Exception e) {
+ throw makeIOException(e);
+ }
+ }
+
+ protected static IOException makeIOException(Throwable e) {
+ if (e instanceof UncheckedExecutionException || e instanceof ExecutionException)
+ e = e.getCause();
+
+ if (e instanceof IOException)
+ return (IOException) e;
+ else if (e.getCause() instanceof IOException)
+ return (IOException) e.getCause();
+ else
+ return new IOException(e);
+ }
+
+ public long size() {
+ return statusCache.size() + childrenCache.size();
+ }
+
+ public int countEvictions() {
+ return countOfEvict;
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/KylinCacheFileSystem.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/KylinCacheFileSystem.java
index 9adfef9e62..176b3716af 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/KylinCacheFileSystem.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/KylinCacheFileSystem.java
@@ -17,30 +17,137 @@
*/
package org.apache.kylin.cache.kylin;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.spark.SparkContext;
import org.apache.spark.TaskContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.spark.sql.SparkSession;
import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
import org.apache.kylin.cache.fs.CacheFileSystemConstants;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class KylinCacheFileSystem extends AbstractCacheFileSystem {
- private static final Logger LOG = LoggerFactory.getLogger(KylinCacheFileSystem.class);
-
/**
* Check whether it needs to cache data on the current executor
*/
@Override
- public boolean isUseLocalCacheForTargetExecs() {
+ protected boolean isUseLocalCacheForCurrentExecutor() {
if (null == TaskContext.get()) {
- LOG.warn("Task Context is null.");
+ log.warn("Task Context is null.");
return false;
}
String localCacheForCurrExecutor = TaskContext.get()
.getLocalProperty(CacheFileSystemConstants.PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES);
- LOG.info("Cache for current executor is {}", localCacheForCurrExecutor);
- return (StringUtils.isNotBlank(localCacheForCurrExecutor) && Boolean.parseBoolean(localCacheForCurrExecutor));
+
+ if (StringUtils.isBlank(localCacheForCurrExecutor))
+ return true; // let the empty default be TRUE
+ else
+ return Boolean.parseBoolean(localCacheForCurrExecutor);
+ }
+
+ @Override
+ protected long getAcceptCacheTime() {
+ return getAcceptCacheTimeLocally();
+ }
+
+ /**
+ * To instruct local cache eviction, sql can contain a hint like
+ * <pre>
+ * select * from xxx /*+ ACCEPT_CACHE_TIME(158176387682000) * / -- no space between * and /
+ * </pre>
+ * <p/>
+ * The value is carried to Spark local property, naming "spark.kylin.local-cache.accept-cache-time",
+ * and then is used by AbstractCacheFileSystem
+ * <p/>
+ * Returns [sql_without_cache_hint, accept_cache_time]
+ */
+ private static final Pattern PTN = Pattern.compile("[(]\\s*([0-9]+)");
+ // returns [sql_with_hint_removed, the_millis_or_null]
+ static String[] extractAcceptCacheTime(String sql) {
+ int cut1 = sql.indexOf("ACCEPT_CACHE_TIME");
+ if (cut1 < 0)
+ return new String[]{sql, null};
+ int cut2 = sql.lastIndexOf("/*", cut1);
+ int cut3 = sql.indexOf("*/", cut1);
+ if (cut2 < 0 || cut3 < 0)
+ return new String[]{sql, null};
+
+ String newSql = sql.substring(0, cut2) + sql.substring(cut3 + 2);
+ String hintStr = sql.substring(cut2, cut3);
+ String millis = null;
+
+ Matcher m = PTN.matcher(hintStr);
+ if (m.find()) {
+ millis = m.group(1);
+ }
+
+ return new String[]{newSql, millis};
+ }
+
+ public static String processAcceptCacheTimeInSql(String sql) {
+ String[] sqlAndAct = extractAcceptCacheTime(sql);
+ setAcceptCacheTimeLocally(sqlAndAct[1]);
+ return sqlAndAct[0];
+ }
+
+ public static void setAcceptCacheTimeLocally(String acceptCacheTime) {
+ setAcceptCacheTimeLocally(acceptCacheTime == null ? System.currentTimeMillis() : Long.parseLong(acceptCacheTime));
+ }
+
+ /**
+ * User can specify that any cache before the required time must be cleared.
+ */
+ public static void setAcceptCacheTimeLocally(long acceptCacheTime) {
+ Preconditions.checkState(SparkSession.getDefaultSession().isDefined());
+ SparkContext sparkContext = SparkSession.getDefaultSession().get().sparkContext();
+ String key = CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME;
+
+ long now = System.currentTimeMillis();
+ if (acceptCacheTime > now + 1000) { // +1000 for some error tolerance of time
+ log.error("Accept-cache-time {} is later than clock time {}. Please check the time sync across machines.", acceptCacheTime, now);
+ acceptCacheTime = now;
+ }
+
+ // if set multiple times, take the max
+ if (sparkContext.getLocalProperty(key) != null) {
+ acceptCacheTime = Math.max(acceptCacheTime, Long.parseLong(sparkContext.getLocalProperty(key)));
+ }
+
+ sparkContext.setLocalProperty(key, Long.toString(acceptCacheTime));
+ }
+
+ public static void clearAcceptCacheTimeLocally() {
+ Preconditions.checkState(SparkSession.getDefaultSession().isDefined());
+ SparkContext sparkContext = SparkSession.getDefaultSession().get().sparkContext();
+ sparkContext.setLocalProperty(CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME, null);
+
+ if (TaskContext.get() != null) { // should not have TaskContext, just in case..
+ TaskContext.get().getLocalProperties().remove(CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME);
+ }
+ }
+
+ public static long getAcceptCacheTimeLocally() {
+ String key = CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME;
+
+ // defaults to get latest data
+ long ret = System.currentTimeMillis();
+
+ if (TaskContext.get() != null) {
+ String prop = TaskContext.get().getLocalProperty(key);
+ if (prop != null)
+ ret = Long.parseLong(prop);
+ } else if (SparkSession.getDefaultSession().isDefined()) {
+ SparkContext ctx = SparkSession.getDefaultSession().get().sparkContext();
+ String prop = ctx.getLocalProperty(key);
+ if (prop != null)
+ ret = Long.parseLong(prop);
+ }
+ return ret;
}
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
index a18e5e6ee9..8e53f5d4d0 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/kylin/OnlyForTestCacheFileSystem.java
@@ -18,16 +18,24 @@
package org.apache.kylin.cache.kylin;
import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
+import org.apache.kylin.cache.fs.ManagerOfCacheFileStatus;
public class OnlyForTestCacheFileSystem extends AbstractCacheFileSystem {
-
-
/**
- * Check whether needs to cache data on the current executor
+ * Check whether it needs to cache data on the current executor
*/
@Override
- public boolean isUseLocalCacheForTargetExecs() {
+ protected boolean isUseLocalCacheForCurrentExecutor() {
return true;
}
+
+ @Override
+ protected long getAcceptCacheTime() {
+ return 0;
+ }
+
+ public ManagerOfCacheFileStatus getStatusCache() {
+ return mStatusCache;
+ }
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityBookKeeping.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityBookKeeping.java
new file mode 100644
index 0000000000..a5c08df654
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityBookKeeping.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.cache.softaffinity;
+
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class SoftAffinityBookKeeping {
+
+ private static final ThreadLocal<LinkedHashMap<String, String>> THREAD = new ThreadLocal<>();
+ private static final int MAX_ENTRIES = 20;
+
+ private SoftAffinityBookKeeping() {
+ }
+
+ public static void recordAsk(String file, String[] locations) {
+ LinkedHashMap<String, String> book = getBook();
+ String loc = String.join("|", locations);
+ String existing = book.putIfAbsent(file, loc);
+ if (existing != null && !existing.contains(loc))
+ book.put(file, existing + "|" + loc);
+ }
+
+ @NotNull
+ private static LinkedHashMap<String, String> getBook() {
+ LinkedHashMap<String, String> book = THREAD.get();
+ if (book == null) {
+ book = new LinkedHashMap<String, String>() {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
+ return size() > MAX_ENTRIES;
+ }
+ };
+ THREAD.set(book);
+ }
+ return book;
+ }
+
+ // returns (file -> locations)
+ public static Map<String, String> audit() {
+ LinkedHashMap<String, String> book = getBook();
+ Map<String, String> ret = ImmutableMap.copyOf(book);
+ book.clear();
+ return ret;
+ }
+
+ public static void logAudits() {
+ List<String> lines = audit().entrySet().stream()
+ .map(e -> e.getKey() + " -> " + e.getValue()).collect(Collectors.toList());
+ log.debug("Past few mappings of location -> executors: \n" + String.join("\n", lines));
+ }
+
+ public static int size() {
+ LinkedHashMap<String, String> book = THREAD.get();
+ return book.size();
+ }
+}
\ No newline at end of file
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityConstants.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityConstants.java
index 68584ad4fd..6c27cbf8a6 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityConstants.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityConstants.java
@@ -22,19 +22,16 @@ public class SoftAffinityConstants {
private SoftAffinityConstants() {
}
- public static final String PARAMS_KEY_SOFT_AFFINITY_ENABLED =
- "spark.kylin.soft-affinity.enabled";
-
+ public static final String PARAMS_KEY_SOFT_AFFINITY_ENABLED = "spark.kylin.soft-affinity.enabled";
public static final boolean PARAMS_KEY_SOFT_AFFINITY_ENABLED_DEFAULT_VALUE = false;
- public static final String PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM =
- "spark.kylin.soft-affinity.replications.num";
+ public static final String PARAMS_KEY_SOFT_AFFINITY_ENABLED_FOR_HIVE = "spark.kylin.soft-affinity.enabled.for-hive";
+ public static final boolean PARAMS_KEY_SOFT_AFFINITY_ENABLED_FOR_HIVE_DEFAULT_VALUE = false;
+ public static final String PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM = "spark.kylin.soft-affinity.replications.num";
public static final int PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM_DEFAULT_VALUE = 2;
// For HDFS Replications
- public static final String PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS =
- "spark.kylin.soft-affinity.min.target-hosts";
-
+ public static final String PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS = "spark.kylin.soft-affinity.min.target-hosts";
public static final int PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS_DEFAULT_VALUE = 1;
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityTextInputFormat.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityTextInputFormat.java
new file mode 100644
index 0000000000..72e242a013
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/softaffinity/SoftAffinityTextInputFormat.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.cache.softaffinity;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.SplitLocationInfo;
+import org.apache.kylin.common.exception.KylinRuntimeException;
+
+import org.apache.kylin.softaffinity.SoftAffinityManager;
+
+/**
+ * A TextInputFormat that reports preferred location based on soft affinity
+ */
+public class SoftAffinityTextInputFormat extends org.apache.hadoop.mapred.TextInputFormat {
+
+ @Override
+ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
+ FileSplit[] splits = (FileSplit[]) super.getSplits(job, numSplits);
+
+ // enable soft affinity, for possible leverage of local cache
+ return SoftAffinityManager.usingSoftAffinity()
+ ? Arrays.stream(splits).map(FileSplitWithCachedLocation::new).toArray(InputSplit[]::new)
+ : splits;
+ }
+
+ public static class FileSplitWithCachedLocation extends FileSplit {
+ org.apache.hadoop.mapreduce.lib.input.FileSplit selfFS;
+ String[] locations;
+
+ @SuppressWarnings("unused") // no-arg constructor for Writable serialization
+ public FileSplitWithCachedLocation() {
+ selfFS = new org.apache.hadoop.mapreduce.lib.input.FileSplit();
+ locations = new String[0];
+ }
+
+ public FileSplitWithCachedLocation(FileSplit s) {
+ try {
+ Field f = FieldUtils.getDeclaredField(FileSplit.class, "fs", true);
+ this.selfFS = (org.apache.hadoop.mapreduce.lib.input.FileSplit) f.get(s);
+ } catch (Exception e) {
+ throw new KylinRuntimeException(e);
+ }
+
+ Path localityBase = s.getPath().getParent();
+ this.locations = SoftAffinityManager.askExecutors(localityBase.toString());
+ }
+
+ @Override
+ public String[] getLocations() {
+ return locations;
+ }
+
+ @Override
+ public SplitLocationInfo[] getLocationInfo() {
+ return Arrays.stream(locations)
+ .map(l -> new SplitLocationInfo(l, false))
+ .toArray(SplitLocationInfo[]::new);
+ }
+
+ @Override
+ public Path getPath() {
+ return selfFS.getPath();
+ }
+
+ @Override
+ public long getStart() {
+ return selfFS.getStart();
+ }
+
+ @Override
+ public long getLength() {
+ return selfFS.getLength();
+ }
+
+ @Override
+ public String toString() {
+ return selfFS.toString();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ selfFS.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ selfFS.readFields(in);
+ }
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/Hadoop3CompaUtil.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/Hadoop3CompaUtil.java
new file mode 100644
index 0000000000..3b36f81ced
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/Hadoop3CompaUtil.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.utils;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.IOUtils;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@SuppressWarnings("unchecked")
+public class Hadoop3CompaUtil {
+
+ private Hadoop3CompaUtil() {}
+
+ public static class RemoteIterators {
+
+ private RemoteIterators() {}
+
+ public static <T> List<T> toList(RemoteIterator<T> source)
+ throws IOException {
+ List<T> l = new ArrayList<>();
+ foreach(source, l::add);
+ return l;
+ }
+
+ public static <T> T[] toArray(RemoteIterator<T> source) throws IOException {
+ return (T[]) toList(source).toArray();
+ }
+
+ public static <T> long foreach(
+ RemoteIterator<T> source,
+ Consumer<? super T> consumer) throws IOException {
+ long count = 0;
+
+ try {
+ while (source.hasNext()) {
+ count++;
+ consumer.accept(source.next());
+ }
+ } finally {
+ if (source instanceof Closeable) {
+ // source is closeable, so close.
+ IOUtils.cleanupWithLogger(log, (Closeable) source);
+ }
+ }
+
+ return count;
+ }
+
+ public static <T> RemoteIterator<T> remoteIteratorFromArray(T[] array) {
+ return new WrappedJavaIterator<>(Arrays.stream(array).iterator());
+ }
+
+ public static <T> RemoteIterator<T> remoteIteratorFromIterator(
+ Iterator<T> iterator) {
+ return new WrappedJavaIterator<>(iterator);
+ }
+ }
+
+ private static final class WrappedJavaIterator<T> implements RemoteIterator<T>, Closeable {
+
+ /**
+ * inner iterator..
+ */
+ private final Iterator<? extends T> source;
+
+ private final Closeable sourceToClose;
+
+
+ /**
+ * Construct from an interator.
+ * @param source source iterator.
+ */
+ private WrappedJavaIterator(Iterator<? extends T> source) {
+ this.source = requireNonNull(source);
+ sourceToClose = new MaybeClose(source);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return source.hasNext();
+ }
+
+ @Override
+ public T next() {
+ return source.next();
+ }
+
+ @Override
+ public String toString() {
+ return "FromIterator{" + source + '}';
+ }
+
+ @Override
+ public void close() throws IOException {
+ sourceToClose.close();
+ }
+ }
+
+ private static final class MaybeClose implements Closeable {
+
+ private Closeable toClose;
+
+ /**
+ * Construct.
+ * @param o object to close.
+ */
+ private MaybeClose(Object o) {
+ this(o, true);
+ }
+
+ /**
+ * Construct -close the object if it is closeable and close==true.
+ * @param o object to close.
+ * @param close should close?
+ */
+ private MaybeClose(Object o, boolean close) {
+ if (close && o instanceof Closeable) {
+ this.toClose = (Closeable) o;
+ } else {
+ this.toClose = null;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (toClose != null) {
+ try {
+ toClose.close();
+ } finally {
+ toClose = null;
+ }
+ }
+ }
+ }
+
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java
index 55cb0f75d8..b25b813b5d 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/cache/utils/ReflectionUtil.java
@@ -21,6 +21,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import org.apache.commons.lang3.ObjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,14 +81,16 @@ public class ReflectionUtil {
*
*/
public static Field getDeclaredField(Object object, String filedName) {
+ NoSuchFieldException noSuchFieldEx = null;
for (Class<?> superClass = object.getClass(); superClass != Object.class; superClass = superClass
.getSuperclass()) {
try {
return superClass.getDeclaredField(filedName);
} catch (NoSuchFieldException e) {
- LOG.error("Can not find field {}", filedName, e);
+ noSuchFieldEx = ObjectUtils.firstNonNull(noSuchFieldEx, e);
}
}
+ LOG.error("Can not find field '{}' in class {}", filedName, object.getClass().getName(), noSuchFieldEx);
return null;
}
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegments.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegments.java
new file mode 100644
index 0000000000..2396293d34
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegments.java
@@ -0,0 +1,612 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.fileseg;
+
+import static org.apache.kylin.metadata.model.TableExtDesc.LOCATION_PROPERTY_KEY;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.compress.utils.Sets;
+import org.apache.curator.shaded.com.google.common.base.Objects;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.KylinRuntimeException;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cube.model.NDataLayout;
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.kylin.metadata.cube.model.NDataflow;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.metadata.model.FilePartitionDesc;
+import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.metadata.model.MultiPartitionDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.PartitionDesc.IPartitionConditionBuilder;
+import org.apache.kylin.metadata.model.PartitionDesc.PartitionType;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.SegmentRange.TimePartitionedSegmentRange;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.spark.SparkContext;
+import org.apache.spark.TaskContext;
+import org.apache.spark.sql.SparkSession;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * A segment partition that is based on source files.
+ * <p/>
+ * Each segment is built from one source file.
+ * <p/>
+ * Term:
+ * - FILE HASH: a (hash) code that represents a file, typically made up of filename, last modified, size etc.
+ */
+@Slf4j
+@SuppressWarnings({"rawtypes"})
+public class FileSegments {
+ // used in SparkContext local property
+ public static String SOURCE_FILTER_KEY = "spark.kylin.fileseg.filter";
+
+ public static String FILE_HASH_SEP = ":";
+
+ /**
+ * Set file segment filter on spark context locally, happens at the DRIVER SIDE.
+ */
+ public static void setFileSegFilterLocally(NDataSegment seg, FileSegRange segRange) {
+ Preconditions.checkState(SparkSession.getDefaultSession().isDefined());
+ SparkContext sparkContext = SparkSession.getDefaultSession().get().sparkContext();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+
+ String sourceFileDir = getSegSourceFileDir(seg, config);
+ String sourceFileHash = String.join(FILE_HASH_SEP, segRange.getFileHash());
+
+ // The sourceFileDir metadata is at "table_exd/data_source_properties/location", value like "s3://maosha-public/liyang/cloud_billings"
+ // The sourceFileHash is a relative path based on the dir, like "cloud_billing_standard_sample.csv"
+
+ sparkContext.setLocalProperty(pathFilterKey(sourceFileDir), sourceFileHash);
+ }
+
+ private static String noEndingSlash(String p) {
+ return p.endsWith("/") ? p.substring(0, p.length() - 1) : p;
+ }
+
+ private static String pathFilterKey(String path) {
+ return SOURCE_FILTER_KEY + "." + noEndingSlash(path);
+ }
+
+ /**
+ * Get file segment filter on spark context locally, happens at the EXECUTOR SIDE.
+ */
+ public static Optional<SourceFilePredicate> getFileSegFilterLocally(String srcDirPath) {
+ String srcFileHash = getSparkLocalProperty(pathFilterKey(srcDirPath), null);
+
+ if (srcFileHash == null)
+ return Optional.empty();
+ else
+ return Optional.of(new SourceFilePredicate(srcDirPath, Sets.newHashSet(srcFileHash.split(FILE_HASH_SEP))));
+ }
+
+ private static String getSparkLocalProperty(String key, @SuppressWarnings("SameParameterValue") String dft) {
+ if (TaskContext.get() != null) {
+ String prop = TaskContext.get().getLocalProperty(key);
+ if (prop != null)
+ return prop;
+ } else if (SparkSession.getDefaultSession().isDefined()) {
+ SparkContext ctx = SparkSession.getDefaultSession().get().sparkContext();
+ String prop = ctx.getLocalProperty(key);
+ if (prop != null)
+ return prop;
+ }
+ return dft;
+ }
+
+ public static void clearFileSegFilterLocally() {
+ if (SparkSession.getDefaultSession().isDefined()) {
+ SparkContext sparkContext = SparkSession.getDefaultSession().get().sparkContext();
+ List<String> keys = sparkContext.getLocalProperties().keySet().stream()
+ .map(Object::toString).collect(Collectors.toList());
+ for (String key : keys) {
+ if (key.startsWith(SOURCE_FILTER_KEY))
+ sparkContext.setLocalProperty(key, null);
+ }
+ }
+ }
+
+ private static String getSegSourceFileDir(NDataSegment seg, KylinConfig config) {
+ NTableMetadataManager tableManager = NTableMetadataManager.getInstance(config, seg.getProject());
+ TableDesc rootFactTable = seg.getDataflow().getModel().getRootFactTable().getTableDesc();
+ TableExtDesc tableExt = tableManager.getTableExtIfExists(rootFactTable);
+ Preconditions.checkNotNull(tableExt);
+ String ret = tableExt.getDataSourceProps().get(LOCATION_PROPERTY_KEY);
+ Preconditions.checkNotNull(ret);
+ return ret;
+ }
+
+ public static String computeFileHash(FileStatus file) {
+ return file.getPath().getName() + "/" + Long.toString(file.getLen(), 36) + "/" + Long.toString(file.getModificationTime(), 36);
+ }
+
+ public static String hashToName(String fileHash) {
+ int cut = fileHash.indexOf('/');
+ if (cut < 0)
+ throw new IllegalArgumentException("Invalid file hash: " + fileHash);
+ return fileHash.substring(0, cut);
+ }
+
+ @SuppressWarnings("BooleanMethodIsAlwaysInverted")
+ public static boolean isFileSegment(NDataSegment seg) {
+ return seg.getSegRange() instanceof FileSegRange;
+ }
+
+ public static void forceFileSegments(String project, String modelId, String storageLocation,
+ @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
+ Optional<List<String>> fileHashOpt,
+ Function<FileSegRange, NDataSegment> segCreator,
+ Function<String, Boolean> segDeleter) {
+ KylinConfig config = NProjectManager.getProjectConfig(project);
+
+ // ensure model has a file based partition
+ NDataModelManager modelManager = config.getManager(project, NDataModelManager.class);
+ NDataModel model = modelManager.getDataModelDesc(modelId);
+ if (model.isBroken())
+ return;
+
+ if (!model.isFilePartitioned()) {
+ FilePartitionDesc newPart = new FilePartitionDesc();
+ newPart.setCubePartitionType(PartitionType.FILE);
+ newPart.setFileStorageLocation(noEndingSlash(storageLocation));
+ newPart.setPartitionConditionBuilderClz(FileSegmentConditionBuilder.class.getName());
+ newPart.setPartitionDateColumn(""); // no use, just for code compatibility
+ newPart.setPartitionDateFormat("yyyy-MM-dd"); // no use, just for code compatibility
+ modelManager.updateDataModel(modelId, (modelCopy) -> modelCopy.setPartitionDesc(newPart));
+ }
+
+ List<String> fileHashs = filterFileHashs(fileHashOpt, project + "/" + model.getAlias(), config);
+ if (CollectionUtils.isEmpty(fileHashs)) {
+ return; // just ensure the model is file-segment enabled, if fileHashs is missing
+ }
+
+ NDataflowManager dfManager = config.getManager(project, NDataflowManager.class);
+ NDataflow df = dfManager.getDataflow(modelId); // yes, dataflow_id == model_id
+ if (df.isBroken())
+ return;
+
+ // mark this data refresh time
+ dfManager.updateDataflow(df.getId(), (copy) -> copy.setLastDataRefreshTime(System.currentTimeMillis()));
+
+ // delete all non-file segments
+ // delete all file-not-exist segments
+ // add all new-file segments
+ List<NDataSegment> origSegments = ImmutableList.copyOf(df.getSegments());
+ Set<String> latestHashs = new HashSet<>(fileHashs);
+ Set<String> toCreateHashs = new LinkedHashSet<>(fileHashs);
+ List<String> toDelSegIds = new ArrayList<>();
+ for (NDataSegment orig : origSegments) {
+ if (!isFileSegment(orig)) {
+ toDelSegIds.add(orig.getId());
+ continue;
+ }
+ List<String> origHash = ((FileSegRange) orig.getSegRange()).getFileHash();
+ if (latestHashs.containsAll(origHash)) {
+ //noinspection SlowAbstractSetRemoveAll
+ toCreateHashs.removeAll(origHash);
+ } else {
+ toDelSegIds.add(orig.getId());
+ }
+ }
+
+ // do the deletion
+ for (String toDelSegId : toDelSegIds) {
+ try {
+ segDeleter.apply(toDelSegId);
+ } catch (Exception ex) {
+ log.error("delete segment failed {}/{}", df.getId(), toDelSegId, ex);
+ }
+ }
+
+ // do the creation
+ FileRangeGenerator gen = new FileRangeGenerator(
+ origSegments.stream()
+ .filter(seg -> !toDelSegIds.contains(seg.getId()))
+ .collect(Collectors.toList()));
+ for (String toCreateHash : toCreateHashs) {
+ try {
+ segCreator.apply(gen.nextFileSegRange(toCreateHash));
+ } catch (Exception ex) {
+ log.error("create segment failed {}/{}", df.getId(), toCreateHash, ex);
+ }
+ }
+
+ // logging
+ ModelFileSegments result = getModelFileSegments(project, modelId);
+ log.info("Sync file segments for {}/{}, {} created, {} deleted, results {}/{}/{}/{}",
+ project, result.model, toCreateHashs.size(), toDelSegIds.size(),
+ result.summary.get(FileSegStatusEnum.LOADED),
+ result.summary.get(FileSegStatusEnum.LOADING),
+ result.summary.get(FileSegStatusEnum.EXIST),
+ result.summary.get(FileSegStatusEnum.WARNING));
+ }
+
+ private static List<String> filterFileHashs(Optional<List<String>> fileHashOpt, String modelForLog, KylinConfig config) {
+ if (!fileHashOpt.isPresent())
+ return Collections.emptyList();
+
+ List<String> hashList = fileHashOpt.get();
+ String successFile = config.getFileSegmentSuccessFile();
+ boolean hasSuccessFile = false;
+
+ List<String> ret = new ArrayList<>(hashList.size());
+ for (String hash : hashList) {
+ String fname = hashToName(hash);
+ if (fname.equals(successFile)) {
+ hasSuccessFile = true; // skip & mark success file
+ continue;
+ }
+ if (fname.startsWith(".") || fname.startsWith("_")) {
+ continue; // file starts with '.' or '_' are hidden files according to hadoop convention
+ }
+
+ ret.add(hash);
+ }
+
+ // a folder without success file is ignored
+ if (successFile != null && !hasSuccessFile) {
+ log.debug("Model {} ignored forceFileSegments() due to missing success file {} from {} detected files", modelForLog, successFile, hashList.size());
+ return Collections.emptyList();
+ }
+
+ return ret;
+ }
+
+ public static ModelFileSegments getModelFileSegments(String project, String modelId) {
+ return getModelFileSegments(project, modelId, false);
+ }
+
+ public static ModelFileSegments getModelFileSegments(String project, String modelId, boolean returnDetails) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ NDataflowManager dfManager = config.getManager(project, NDataflowManager.class);
+ NDataflow df = dfManager.getDataflow(modelId); // yes, dataflow id == model id
+ Preconditions.checkState(df.getModel().isFilePartitioned());
+
+ // collect file->[segment status], note a file may have two segments, one is READY, another is NEW (building)
+ Map<String, Set<NDataSegment>> fileToSegs = new LinkedHashMap<>();
+ for (NDataSegment seg : df.getSegments()) {
+ SegmentRange range = seg.getSegRange();
+ if (!(range instanceof FileSegRange))
+ continue; // quote impossible, though
+
+ FileSegRange fileRange = (FileSegRange) range;
+ for (String fileHash : fileRange.fileHash) {
+ fileToSegs.compute(fileHash,
+ (hash, set) -> {
+ if (set == null)
+ set = new HashSet<>();
+ set.add(seg);
+ return set;
+ });
+ }
+ }
+
+ List<FileSegStatus> segStatuses = Lists.newArrayList();
+ int loading = 0;
+ int loaded = 0;
+ int warning = 0;
+ int exist = 0;
+ for (Entry<String, Set<NDataSegment>> entry : fileToSegs.entrySet()) {
+ String fileHash = entry.getKey();
+ Set<NDataSegment> segs = entry.getValue();
+ FileSegStatusEnum status;
+ if (containsSegLike(segs, SegmentStatusEnum.NEW)) {
+ status = FileSegStatusEnum.LOADING;
+ loading++;
+ } else if (containsSegWithIndex(segs)) {
+ status = FileSegStatusEnum.LOADED;
+ loaded++;
+ } else if (containsSegLike(segs, SegmentStatusEnum.WARNING)) {
+ status = FileSegStatusEnum.WARNING;
+ warning++;
+ } else {
+ status = FileSegStatusEnum.EXIST;
+ exist++;
+ }
+
+ if (returnDetails) {
+ segStatuses.add(new FileSegStatus(fileHash, status, 0));
+ }
+ }
+
+ Map<FileSegStatusEnum, Integer> summary = ImmutableMap.of(
+ FileSegStatusEnum.LOADED, loaded,
+ FileSegStatusEnum.LOADING, loading,
+ FileSegStatusEnum.EXIST, exist,
+ FileSegStatusEnum.WARNING, warning);
+
+ // check if any index is being built (not new segment, but new index)
+ boolean isBuildingIndex = guessIsBuildingIndex(df);
+
+ return new ModelFileSegments(project, df.getModelAlias(),
+ df.getLastDataRefreshTime(), df.getIndexPlan().getLastModified(),
+ loading > 0, isBuildingIndex, false,
+ summary, segStatuses);
+ }
+
+ public static boolean guessIsBuildingIndex(NDataflow df) {
+ Set<Long> plannedLayouts = df.getIndexPlan().getAllLayouts().stream().map(layout -> layout.getId()).collect(Collectors.toSet());
+ for (NDataSegment seg : df.getSegments()) {
+ if (seg.getStatus() == SegmentStatusEnum.NEW)
+ continue; // ignore new data loading
+ if ("FULL_BUILD".equals(seg.getName()))
+ break; // for file-seg-model, a FULL_BUILD seg means empty file folder, ignore
+ Map<Long, NDataLayout> layouts = seg.getLayoutsMap();
+ if (layouts == null || !layouts.keySet().containsAll(plannedLayouts)) {
+ // some segment is missing planned layouts
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean containsSegWithIndex(Set<NDataSegment> segs) {
+ return segs.stream().anyMatch(seg -> seg.getLayoutSize() > 0);
+ }
+
+ private static boolean containsSegLike(Set<NDataSegment> segs, SegmentStatusEnum status) {
+ return segs.stream().anyMatch(seg -> seg.getStatus() == status);
+ }
+
+ public static String makeSyncFileSegSql(String selectSql) {
+ return "select count(*) _SYNC_FILE_SEGMENTS_ from ( " + selectSql + " )";
+ }
+
+ public static String makeSyncFileSegSql(List<NDataModel> models) {
+ List<String> factTables = models.stream()
+ .map(m -> m.getRootFactTableName())
+ .distinct()
+ .collect(Collectors.toList());
+
+ return factTables.stream()
+ .map(tbl -> "select count(*) _SYNC_FILE_SEGMENTS_ from " + tbl)
+ .collect(Collectors.joining(" union all "));
+ }
+
+ public static boolean isSyncFileSegSql(String sql) {
+ return sql.contains("_SYNC_FILE_SEGMENTS_");
+ }
+
+ public static List<NDataModel> findModelsOfFileSeg(String project, List<String> modelAliasOrFactTables) {
+ if (modelAliasOrFactTables == null || modelAliasOrFactTables.isEmpty())
+ return ImmutableList.of();
+
+ List<NDataModel> models = listModelsOfFileSeg(project);
+ return models.stream()
+ .filter(m -> modelAliasOrFactTables.stream()
+ .anyMatch(inp -> m.getAlias().equalsIgnoreCase(inp) || m.getRootFactTableName().equalsIgnoreCase(inp)))
+ .collect(Collectors.toList());
+ }
+
+ public static List<NDataModel> listModelsOfFileSeg(String project) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ NDataflowManager dfManager = config.getManager(project, NDataflowManager.class);
+
+ List<NDataModel> models = dfManager.listUnderliningDataModels();
+ return models.stream()
+ .filter(m -> m.isFilePartitioned() && ((FilePartitionDesc) m.getPartitionDesc()).getFileStorageLocation() != null)
+ .collect(Collectors.toList());
+ }
+
+ public enum FileSegStatusEnum {
+ LOADED, LOADING, EXIST, WARNING
+ }
+
+ @Data
+ @NoArgsConstructor
+ @AllArgsConstructor
+ public static class ModelFileSegments {
+
+ public static ModelFileSegments broken(String project, String modelAlias) {
+ ModelFileSegments ret = new ModelFileSegments();
+ ret.setProject(project);
+ ret.setModel(modelAlias);
+ ret.setBroken(true);
+ return ret;
+ }
+
+ @JsonProperty("project")
+ private String project;
+ @JsonProperty("model")
+ private String model;
+ @JsonProperty("last_data_refresh_time")
+ private long lastDataRefreshTime;
+ @JsonProperty("last_index_refresh_time")
+ private long lastIndexRefreshTime;
+ @JsonProperty("is_loading_data")
+ private boolean isLoadingData;
+ @JsonProperty("is_building_index")
+ private boolean isBuildingIndex;
+ @JsonProperty("is_broken")
+ private boolean isBroken;
+ @JsonProperty("summary")
+ private Map<FileSegStatusEnum, Integer> summary;
+ @JsonProperty("segments")
+ private List<FileSegStatus> segments;
+ }
+
+ @Data
+ @NoArgsConstructor
+ @AllArgsConstructor
+ public static class FileSegStatus {
+ @JsonProperty("file_hash")
+ private String fileHash;
+ @JsonProperty("status")
+ private FileSegStatusEnum status;
+ @JsonProperty("progress_bar")
+ private Integer progressBar; // progress bar of 0-100
+ }
+
+ private static class FileRangeGenerator {
+ final SimpleDateFormat dateFormat;
+ int nextStartYear = 2000;
+
+ FileRangeGenerator(List<NDataSegment> origSegments) {
+ dateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.getDefault(Locale.Category.FORMAT));
+ dateFormat.setTimeZone(TimeZone.getDefault());
+
+ Optional<Long> maxEnd = origSegments.stream()
+ .map(seg -> (Long) seg.getSegRange().getEnd())
+ .reduce(Math::max);
+ maxEnd.ifPresent(millis -> nextStartYear = yearOf(millis));
+
+ // prevention of time range overflow
+ if (nextStartYear > 22000) {
+ nextStartYear = 1970;
+ log.error("File segments time range running out?? SegIds: "
+ + origSegments.stream().map(seg -> seg.getId()).collect(Collectors.joining(",")));
+ }
+ }
+
+ private int yearOf(long millis) {
+ String format = dateFormat.format(new Date(millis));
+ int cut = format.indexOf('-');
+ return Integer.parseInt(format.substring(0, cut));
+ }
+
+ private long millisOf(int year) {
+ try {
+ return dateFormat.parse(year + "-01-01").getTime();
+ } catch (ParseException e) {
+ throw new KylinRuntimeException(e); // unreachable
+ }
+ }
+
+ public FileSegRange nextFileSegRange(String toCreateHash) {
+ FileSegRange ret = new FileSegRange();
+ ret.setStart(millisOf(nextStartYear++));
+ ret.setEnd(millisOf(nextStartYear));
+ ret.setFileHash(ImmutableList.of(toCreateHash));
+ return ret;
+ }
+ }
+
+ @RequiredArgsConstructor
+ public static class SourceFilePredicate {
+
+ final String srcDir;
+ final Set<String> srcFileHash;
+
+ public boolean checkFile(LocatedFileStatus f) {
+ if (f.isDirectory())
+ return true;
+
+ String thisFile = computeFileHash(f);
+ boolean ok = srcFileHash.contains(thisFile);
+ if (!ok) {
+ log.trace("filtered out file {}, the filter is {}", thisFile, srcFileHash);
+ }
+ return ok;
+ }
+ }
+
+ // used by reflection of full class name
+ public static class FileSegmentConditionBuilder implements IPartitionConditionBuilder, Serializable {
+ @Override
+ public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) {
+
+ setFileSegFilterLocally((NDataSegment) seg, (FileSegRange) segRange);
+
+ return "1=1";
+ }
+
+ @Override
+ public String buildMultiPartitionCondition(PartitionDesc partDesc, MultiPartitionDesc multiPartDesc, LinkedList<Long> partitionIds, ISegment seg, SegmentRange segRange) {
+ return "1=1";
+ }
+ }
+
+ /**
+ * This SHALL BETTER implement SegmentRange DIRECTLY.
+ * But TimePartitionedSegmentRange is hard-coded at too many places, extending it is a temp workaround.
+ */
+ @Getter
+ @Setter
+ public static class FileSegRange extends TimePartitionedSegmentRange {
+ @JsonProperty("file_hash")
+ private List<String> fileHash;
+
+ @Override
+ protected void checkSameType(SegmentRange<Long> o) {
+ Preconditions.checkNotNull(o);
+ }
+
+ @Override
+ public SegmentRange<Long> coverWith(SegmentRange<Long> other) {
+ FileSegRange o = (FileSegRange) other;
+ FileSegRange ret = new FileSegRange();
+ ret.start = Math.min(this.start, o.start);
+ ret.end = Math.max(this.end, o.end);
+ ret.fileHash = new ArrayList<>();
+ ret.fileHash.addAll(Objects.firstNonNull(this.fileHash, Collections.emptyList()));
+ ret.fileHash.addAll(Objects.firstNonNull(o.fileHash, Collections.emptyList()));
+ return ret;
+ }
+
+ @Override
+ public String proposeSegmentName() {
+ if (fileHash == null || fileHash.isEmpty())
+ return null;
+ else
+ return String.join("|", fileHash);
+ }
+ }
+
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegmentsDetector.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegmentsDetector.java
new file mode 100644
index 0000000000..349f95560b
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/fileseg/FileSegmentsDetector.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.fileseg;
+
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.metadata.model.FilePartitionDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class FileSegmentsDetector {
+
+ private static final ThreadLocal<FileSegmentsDetector> local = new ThreadLocal<>();
+
+ public static void startLocally(String project) {
+ local.set(new FileSegmentsDetector(project));
+ }
+
+ public static void onObserveFileSystemListFileStatus(Path p, LocatedFileStatus[] content) {
+ if (p == null || content == null)
+ return;
+
+ // check we are at the driver side; executor side don't have query context
+ QueryContext queryContext = QueryContext.current();
+ if (queryContext == null)
+ return;
+
+ // check we are detecting
+ FileSegmentsDetector detector = local.get();
+ if (detector == null)
+ return;
+
+ detector._onObserveFileSystemListFileStatus(p, content);
+ }
+
+ public static void report(Consumer<Finding> findingConsumer) {
+ FileSegmentsDetector detector = local.get();
+ if (detector == null || findingConsumer == null)
+ return;
+
+ try {
+ detector._report(findingConsumer);
+ } catch (Exception ex) {
+ log.error("Tolerate error reporting detected file segments", ex);
+ }
+ }
+
+ public static void endLocally() {
+ local.remove();
+ }
+
+ @RequiredArgsConstructor
+ public static class Finding {
+ public final String project;
+ public final String modelId;
+ public final String storageLocation;
+ public final List<String> fileHashs;
+ }
+
+ // =======================================
+
+ private final String project;
+
+ private boolean inited;
+ private Map<String, Set<String>> findings;
+
+
+ public FileSegmentsDetector(String project) {
+ this.project = project;
+ }
+
+ private void lazyInit() {
+ if (inited)
+ return;
+
+ findings = new HashMap<>();
+ inited = true;
+ }
+
+ /*
+ * Note: Should only push-down query reaches here. Cube query is NOT intercepted by AbstractCacheFileSystem
+ */
+ private void _onObserveFileSystemListFileStatus(Path p, LocatedFileStatus[] content) {
+ lazyInit();
+
+ Set<String> set = findings.computeIfAbsent(p.toString(), path -> new LinkedHashSet<>());
+ for (LocatedFileStatus file : content) {
+ set.add(FileSegments.computeFileHash(file));
+ }
+ }
+
+ private void _report(Consumer<Finding> findingConsumer) {
+ List<NDataModel> models = FileSegments.listModelsOfFileSeg(project);
+ for (NDataModel model : models) {
+ String loc = ((FilePartitionDesc) model.getPartitionDesc()).getFileStorageLocation();
+ if (loc == null || !findings.containsKey(loc))
+ continue;
+
+ Set<String> fileHashs = findings.get(loc);
+ try {
+ findingConsumer.accept(new Finding(project, model.getId(), loc, ImmutableList.copyOf(fileHashs)));
+ } catch (Exception ex) {
+ log.error("Tolerate error while model {} consuming detected file segments {}", project + "/" + model.getAlias(), fileHashs, ex);
+ }
+ }
+ }
+
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/hive/serde2/lazy/LazyQuoteAwareSerDe.java b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/hive/serde2/lazy/LazyQuoteAwareSerDe.java
new file mode 100644
index 0000000000..3f08429458
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/java/org/apache/kylin/hive/serde2/lazy/LazyQuoteAwareSerDe.java
@@ -0,0 +1,795 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.hive.serde2.lazy;
+
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeSpec;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyDate;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
+import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazyString;
+import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
+import org.apache.hadoop.hive.serde2.lazy.LazyTimestamp;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDateObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hive.common.util.TimestampParser;
+import org.apache.kylin.common.exception.KylinRuntimeException;
+
+/**
+ * This class contains ALL of our enhancement to Hive's LazySimpleSerDe, for <a href="https://github.com/Kyligence/Diting/blob/release-2.0.0/insight-core/src/main/java/org/apache/hadoop/hive/serde2/lazy/LazyQuoteAwareSerDe.java">
+ * a copy of this file</a> is placed into project Diting, module insight-core, to
+ * ensure the consistent behavior of uploading and querying a CSV file.
+ * <br/>
+ * One big file is easier to copy over.
+ * <br/>
+ * TEST PATH (from UT to IT):
+ * - LazyQuoteAwareSerDeTest
+ * - CsvLocalInferUtilTest (in Diting)
+ * - HiveSoftAffinityAndLocalCacheTest (in KAP)
+ * - in Zen GUI, upload a CSV, make a metric, and query it
+ */
+
+@SerDeSpec(schemaProps = {
+ serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES,
+ serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM,
+ serdeConstants.SERIALIZATION_FORMAT, serdeConstants.SERIALIZATION_NULL_FORMAT,
+ serdeConstants.SERIALIZATION_ESCAPE_CRLF,
+ serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST,
+ serdeConstants.ESCAPE_CHAR, serdeConstants.QUOTE_CHAR,
+ serdeConstants.SERIALIZATION_ENCODING,
+ LazySerDeParameters.SERIALIZATION_EXTEND_NESTING_LEVELS,
+ LazySerDeParameters.SERIALIZATION_EXTEND_ADDITIONAL_NESTING_LEVELS
+})
+public class LazyQuoteAwareSerDe extends LazySimpleSerDe {
+
+ public static List<String[]> parseCsvLines(List<String> lines) {
+ return parseCsvLines(lines, ",", "\\", "\"");
+ }
+
+ public static List<String[]> parseCsvLines(List<String> lines, String separator, String escape, String quote) {
+ if (lines.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ int colCountEst = lines.stream()
+ .map(l -> l.chars().filter(c -> c == separator.charAt(0)).count() + 1)
+ .max(Long::compare)
+ .orElse(0L).intValue();
+
+ try {
+ Properties tbl = new Properties();
+ tbl.put("field.delim", separator);
+ tbl.put("escape.delim", escape);
+ tbl.put("quote.delim", quote);
+ tbl.put("columns", IntStream.range(0, colCountEst).boxed().map(i -> "col" + i)
+ .collect(Collectors.joining(",")));
+ tbl.put("columns.types", StringUtils.repeat("string", ":", colCountEst));
+
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ serde.initialize(new Configuration(), tbl);
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+
+ List<String[]> ret = new ArrayList<>();
+ int nonNullColLen = 0;
+ for (String line : lines) {
+ LazyStruct struct = (LazyStruct) serde.deserialize(new Text(line));
+ String[] row = new String[fields.size()];
+ for (int i = 0; i < fields.size(); i++) {
+ //noinspection unchecked
+ LazyPrimitive<ObjectInspector, Writable> val =
+ (LazyPrimitive<ObjectInspector, Writable>) soi.getStructFieldData(struct, fields.get(i));
+ row[i] = val == null ? null : val.getWritableObject().toString();
+ if (val != null) {
+ nonNullColLen = Math.max(nonNullColLen, i + 1);
+ }
+ }
+ ret.add(row);
+ }
+
+ // cut off tailing all-null columns
+ if (nonNullColLen < fields.size()) {
+ for (int i = 0; i < ret.size(); i++) {
+ String[] row = ret.get(i);
+ ret.set(i, Arrays.copyOf(row, nonNullColLen));
+ }
+ }
+
+ return ret;
+
+ } catch (Exception ex) {
+ throw new KylinRuntimeException(ex);
+ }
+ }
+
+ // =============================================
+
+ private QuoteAwareStruct myStruct;
+
+ public LazyQuoteAwareSerDe() throws SerDeException {
+ // nothing to do
+ }
+
+ @Override
+ public void initialize(Configuration job, Properties tbl) throws SerDeException {
+ super.initialize(job, tbl);
+
+ byte quote = '"';
+ if (tbl.containsKey(serdeConstants.QUOTE_CHAR)) {
+ quote = (byte) tbl.getProperty(serdeConstants.QUOTE_CHAR).charAt(0);
+ }
+
+ boolean numberForceRead = tbl.containsKey("number.force")
+ && !"false".equals(tbl.getProperty("number.force"));
+
+ // overwrite the LazyStruct field to be QuoteAwareStruct
+ myStruct = new QuoteAwareStruct(
+ (LazySimpleStructObjectInspector) getObjectInspector(),
+ quote,
+ numberForceRead);
+
+ try {
+ FieldUtils.writeField(this, "cachedLazyStruct", myStruct, true);
+ } catch (Exception e) {
+ throw new KylinRuntimeException(e);
+ }
+
+ }
+
+ public boolean hasExtraFieldWarn() {
+ return myStruct.selfExtraFieldWarned;
+ }
+
+ public boolean hasMissingFieldWarn() {
+ return myStruct.selfMissingFieldWarned;
+ }
+
+ public static class QuoteAwareStruct extends LazyStruct {
+ private final boolean quoteEnabled;
+ private final byte quote;
+ private final boolean numberForceRead;
+ private boolean[] isFieldQuoted;
+
+ // copied from LazyStruct
+ boolean selfParsed;
+ int[] selfStartPosition;
+ LazyObjectBase[] selfFields;
+ boolean[] selfFieldInited;
+ boolean selfMissingFieldWarned = false;
+ boolean selfExtraFieldWarned = false;
+
+ public QuoteAwareStruct(LazySimpleStructObjectInspector oi, Byte quote, boolean numberForceRead) {
+ super(oi);
+ this.quoteEnabled = quote != null;
+ this.quote = quote == null ? 0 : quote;
+ this.numberForceRead = numberForceRead;
+ }
+
+ /*
+ * Called only once for each field.
+ */
+ @Override
+ protected LazyObjectBase createLazyField(int fieldId, StructField fieldRef) {
+ ObjectInspector foi = fieldRef.getFieldObjectInspector();
+ if (foi instanceof LazyBooleanObjectInspector) {
+ ((LazyBooleanObjectInspector) foi).setExtendedLiteral(true);
+ }
+
+ if (foi instanceof LazyStringObjectInspector) {
+ return quoteEnabled
+ ? new LazyQuoteAwareString((LazyStringObjectInspector) foi, quote)
+ : new LazyString((LazyStringObjectInspector) foi);
+ } else if (foi instanceof LazyDateObjectInspector) {
+ return new LazyDateEx((LazyDateObjectInspector) foi);
+ } else if (foi instanceof LazyTimestampObjectInspector) {
+ return new LazyTimestampEx((LazyTimestampObjectInspector) foi);
+ } else if (foi instanceof LazyDoubleObjectInspector) {
+ return new LazyDoubleEx((LazyDoubleObjectInspector) foi, numberForceRead);
+ } else if (foi instanceof LazyFloatObjectInspector) {
+ return new LazyFloatEx((LazyFloatObjectInspector) foi, numberForceRead);
+ } else if (foi instanceof LazyIntObjectInspector) {
+ return new LazyIntEx((LazyIntObjectInspector) foi, numberForceRead);
+ } else if (foi instanceof LazyLongObjectInspector) {
+ return new LazyLongEx((LazyLongObjectInspector) foi, numberForceRead);
+ } else {
+ return LazyFactory.createLazyObject(foi);
+ }
+ }
+
+ /*
+ * Resets data in this structure object. Called at the beginning of deserializing a new row.
+ */
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ super.init(bytes, start, length);
+ selfParsed = false;
+ }
+
+ @Override
+ public Object getField(int fieldId) {
+ if (!selfParsed) {
+ parse0();
+ }
+ return uncheckedGetField0(fieldId);
+ }
+
+ private void parse0() {
+
+ byte separator = oi.getSeparator();
+ boolean lastColumnTakesRest = oi.getLastColumnTakesRest();
+ boolean isEscaped = oi.isEscaped();
+ byte escapeChar = oi.getEscapeChar();
+
+ initFieldQuoted();
+
+ int structByteEnd = start + length;
+ int fieldId = 0;
+ int fieldByteBegin = start;
+ int fieldByteEnd = start;
+ byte[] bytes = this.bytes.getData();
+
+ // Go through all bytes in the byte[]
+ while (fieldByteEnd <= structByteEnd) {
+ // check if is the end of a field, note the quote begin and end must match
+ boolean isSep = checkEndField(separator, structByteEnd, fieldByteBegin, fieldByteEnd, bytes);
+ if (fieldByteEnd == structByteEnd || isSep) {
+ // Reached the end of a field?
+ fieldByteEnd = getFieldByteEnd(lastColumnTakesRest, structByteEnd, fieldId, fieldByteEnd);
+ // Is quoted?
+ checkFieldQuoted(fieldId, fieldByteBegin, fieldByteEnd, bytes);
+ // Mark field boundary
+ selfStartPosition[fieldId] = fieldByteBegin;
+ fieldId++;
+ if (fieldId == selfFields.length || fieldByteEnd == structByteEnd) {
+ // All fields have been parsed, or bytes have been parsed.
+ // We need to set the startPosition of fields.length to ensure we
+ // can use the same formula to calculate the length of each field.
+ // For missing fields, their starting positions will all be the same,
+ // which will make their lengths to be -1 and uncheckedGetField will
+ // return these fields as NULLs.
+ for (int i = fieldId; i <= selfFields.length; i++) {
+ selfStartPosition[i] = fieldByteEnd + 1;
+ }
+ break;
+ }
+ fieldByteBegin = fieldByteEnd + 1;
+ fieldByteEnd++;
+ } else {
+ if (isEscaped && bytes[fieldByteEnd] == escapeChar
+ && fieldByteEnd + 1 < structByteEnd) {
+ // ignore the char after escape_char
+ fieldByteEnd += 2;
+ } else {
+ fieldByteEnd++;
+ }
+ }
+ }
+
+ // Extra bytes at the end?
+ checkExtraBytesAtTheEnd(structByteEnd, fieldByteEnd);
+
+ // Missing fields?
+ checkMissingFields(fieldId);
+
+ Arrays.fill(selfFieldInited, false);
+ selfParsed = true;
+ }
+
+ private int getFieldByteEnd(boolean lastColumnTakesRest, int structByteEnd, int fieldId, int fieldByteEnd) {
+ if (lastColumnTakesRest && fieldId == selfFields.length - 1) {
+ fieldByteEnd = structByteEnd;
+ }
+ return fieldByteEnd;
+ }
+
+ private void initFieldQuoted() {
+ if (selfFields == null) {
+ initLazyFields2(oi.getAllStructFieldRefs());
+ isFieldQuoted = new boolean[selfFields.length];
+ }
+ Arrays.fill(isFieldQuoted, false);
+ }
+
+ private void checkFieldQuoted(int fieldId, int fieldByteBegin, int fieldByteEnd, byte[] bytes) {
+ if (quoteEnabled) {
+ isFieldQuoted[fieldId] = (fieldByteEnd - fieldByteBegin >= 2
+ && bytes[fieldByteBegin] == quote && bytes[fieldByteEnd - 1] == quote);
+ }
+ }
+
+ private void checkMissingFields(int fieldId) {
+ if (!selfMissingFieldWarned && fieldId < selfFields.length) {
+ selfMissingFieldWarned = true;
+ }
+ }
+
+ private void checkExtraBytesAtTheEnd(int structByteEnd, int fieldByteEnd) {
+ if (!selfExtraFieldWarned && fieldByteEnd < structByteEnd) {
+ selfExtraFieldWarned = true;
+ }
+ }
+
+ private boolean checkEndField(byte separator, int structByteEnd, int fieldByteBegin, int fieldByteEnd, byte[] bytes) {
+ boolean isSep = fieldByteEnd == structByteEnd || bytes[fieldByteEnd] == separator;
+ if (isSep && quoteEnabled
+ && (fieldByteBegin < structByteEnd && bytes[fieldByteBegin] == quote) // it begins with "
+ && (fieldByteEnd < fieldByteBegin + 2 || bytes[fieldByteEnd - 1] != quote)) { // but ending is not
+ isSep = false;
+ }
+ return isSep;
+ }
+
+ private void initLazyFields2(List<? extends StructField> fieldRefs) {
+ selfFields = new LazyObjectBase[fieldRefs.size()];
+ for (int i = 0; i < selfFields.length; i++) {
+ try {
+ selfFields[i] = createLazyField(i, fieldRefs.get(i));
+ } catch (Exception e) {
+ throw new KylinRuntimeException(e);
+ }
+ }
+ selfFieldInited = new boolean[selfFields.length];
+ // Extra element to make sure we have the same formula to compute the
+ // length of each element of the array.
+ selfStartPosition = new int[selfFields.length + 1];
+ }
+
+ private Object uncheckedGetField0(int fieldId) {
+ if (selfFieldInited[fieldId]) {
+ return selfFields[fieldId].getObject();
+ }
+ selfFieldInited[fieldId] = true;
+
+ int fieldByteBegin = selfStartPosition[fieldId];
+ int fieldLength = selfStartPosition[fieldId + 1] - selfStartPosition[fieldId] - 1;
+ if (isFieldQuoted[fieldId]) {
+ fieldByteBegin++;
+ fieldLength -= 2;
+ }
+ if (isNull(oi.getNullSequence(), bytes, fieldByteBegin, fieldLength)) {
+ selfFields[fieldId].setNull();
+ } else {
+ if (isFieldQuoted[fieldId] && selfFields[fieldId] instanceof LazyQuoteAwareString) {
+ ((LazyQuoteAwareString) selfFields[fieldId]).initQuoted(bytes, fieldByteBegin, fieldLength);
+ } else {
+ selfFields[fieldId].init(bytes, fieldByteBegin, fieldLength);
+ }
+ }
+ return selfFields[fieldId].getObject();
+ }
+
+ private transient List<Object> cachedList;
+
+ @Override
+ public List<Object> getFieldsAsList() {
+ if (!selfParsed) {
+ parse0();
+ }
+ if (cachedList == null) {
+ cachedList = new ArrayList<>();
+ } else {
+ cachedList.clear();
+ }
+ for (int i = 0; i < selfFields.length; i++) {
+ cachedList.add(uncheckedGetField0(i));
+ }
+ return cachedList;
+ }
+ }
+
+ public static class LazyQuoteAwareString extends LazyString {
+ private final byte quote;
+
+ public LazyQuoteAwareString(LazyStringObjectInspector oi, byte quote) {
+ super(oi);
+ this.quote = quote;
+ }
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ if (oi.isEscaped()) {
+ byte escapeChar = oi.getEscapeChar();
+ byte[] inputBytes = bytes.getData();
+ LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data);
+ } else {
+ // if the data is not escaped, simply copy the data.
+ data.set(bytes.getData(), start, length);
+ }
+ isNull = false;
+ }
+
+ public void initQuoted(ByteArrayRef bytes, int fieldByteBegin, int fieldLength) {
+ init(bytes, fieldByteBegin, fieldLength);
+
+ // transform double quotes "" into one quote "
+ transformDoubleQuotes();
+ }
+
+ private void transformDoubleQuotes() {
+ if (isNull) {
+ return;
+ }
+
+ byte[] bytes = data.getBytes();
+ int len = data.getLength();
+ if (len < 2) {
+ return;
+ }
+
+ // first check if there is double quotes
+ int i = 0;
+ while (i < len - 1) {
+ if (bytes[i] == quote && bytes[i + 1] == quote) {
+ break;
+ }
+ i++;
+ }
+ if (i == len - 1) { // no double quotes
+ return; // quick return
+ }
+
+ // found double quotes, start to transform
+ int target = i;
+ while (i < len) {
+ if (i < len - 1 && bytes[i] == quote && bytes[i + 1] == quote) {
+ i++;
+ }
+ bytes[target++] = bytes[i++];
+ }
+
+ // inside Text.set(), System.arraycopy will mind the copy on the same object
+ data.set(bytes, 0, target);
+ }
+ }
+
+ /**
+ * The general contract of the LazyXxxPrimitive implementations:
+ * <br/>
+ * - Make the best effort to extract the primitive value, like skipping any letter to parse an integer
+ * - Never throw exception
+ * - In case of error, set isNull && call logExceptionMessage()
+ */
+
+ public static class LazyDateEx extends LazyDate {
+ public LazyDateEx(LazyDateObjectInspector oi) {
+ super(oi);
+ }
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ String s;
+ try {
+ s = Text.decode(bytes.getData(), start, length);
+ data.set(PrimitiveParser.parseDate(s));
+ isNull = false;
+ } catch (Exception e) {
+ isNull = true;
+ logExceptionMessage(bytes, start, length, "DATE");
+ }
+ }
+ }
+
+ public static class LazyTimestampEx extends LazyTimestamp {
+ public LazyTimestampEx(LazyTimestampObjectInspector oi) {
+ super(oi);
+ }
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ String s;
+ s = new String(bytes.getData(), start, length, StandardCharsets.US_ASCII);
+
+ Timestamp t = null;
+ if (s.compareTo("NULL") == 0) {
+ isNull = true;
+ logExceptionMessage(bytes, start, length, "TIMESTAMP");
+ } else {
+ try {
+ t = PrimitiveParser.parseTimestamp(s);
+ isNull = false;
+ } catch (IllegalArgumentException e) {
+ isNull = true;
+ logExceptionMessage(bytes, start, length, "TIMESTAMP");
+ }
+ }
+ data.set(t);
+ }
+ }
+
+ abstract public static class LazyGenericNumber<OI extends ObjectInspector, T extends Writable> extends LazyPrimitive<OI, T> {
+ private final boolean forceRead;
+
+ protected LazyGenericNumber(OI oi, boolean forceRead) {
+ super(oi);
+ this.data = newEmptyWritable();
+ this.forceRead = forceRead;
+ }
+
+ abstract protected T newEmptyWritable();
+
+ abstract protected void parseAndSet(String str, T data, boolean force);
+
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ String str;
+ if (!LazyUtils.isNumberMaybe(bytes.getData(), start, length)) {
+ isNull = true;
+ return;
+ }
+ try {
+ str = Text.decode(bytes.getData(), start, length);
+ parseAndSet(str, data, forceRead);
+ isNull = false;
+ } catch (RuntimeException | CharacterCodingException e) {
+ isNull = true;
+ logExceptionMessage(bytes, start, length, "NUMBER");
+ }
+ }
+ }
+
+ public static class LazyDoubleEx extends LazyGenericNumber<LazyDoubleObjectInspector, DoubleWritable> {
+ public LazyDoubleEx(LazyDoubleObjectInspector lazyDoubleObjectInspector, boolean forceRead) {
+ super(lazyDoubleObjectInspector, forceRead);
+ }
+
+ @Override
+ protected DoubleWritable newEmptyWritable() {
+ return new DoubleWritable();
+ }
+
+ @Override
+ protected void parseAndSet(String str, DoubleWritable data, boolean force) {
+ data.set(PrimitiveParser.parseDouble(str, force));
+ }
+ }
+
+ public static class LazyFloatEx extends LazyGenericNumber<LazyFloatObjectInspector, FloatWritable> {
+ public LazyFloatEx(LazyFloatObjectInspector lazyFloatObjectInspector, boolean forceRead) {
+ super(lazyFloatObjectInspector, forceRead);
+ }
+
+ @Override
+ protected FloatWritable newEmptyWritable() {
+ return new FloatWritable();
+ }
+
+ @Override
+ protected void parseAndSet(String str, FloatWritable data, boolean force) {
+ data.set(PrimitiveParser.parseFloat(str, force));
+ }
+ }
+
+ public static class LazyIntEx extends LazyGenericNumber<LazyIntObjectInspector, IntWritable> {
+ public LazyIntEx(LazyIntObjectInspector lazyIntegerObjectInspector, boolean forceRead) {
+ super(lazyIntegerObjectInspector, forceRead);
+ }
+
+ @Override
+ protected IntWritable newEmptyWritable() {
+ return new IntWritable();
+ }
+
+ @Override
+ protected void parseAndSet(String str, IntWritable data, boolean force) {
+ data.set(PrimitiveParser.parseInteger(str, force));
+ }
+ }
+
+ public static class LazyLongEx extends LazyGenericNumber<LazyLongObjectInspector, LongWritable> {
+ public LazyLongEx(LazyLongObjectInspector lazyLongObjectInspector, boolean forceRead) {
+ super(lazyLongObjectInspector, forceRead);
+ }
+
+ @Override
+ protected LongWritable newEmptyWritable() {
+ return new LongWritable();
+ }
+
+ @Override
+ protected void parseAndSet(String str, LongWritable data, boolean force) {
+ data.set(PrimitiveParser.parseLong(str, force));
+ }
+ }
+
+ public static class PrimitiveParser {
+
+ private PrimitiveParser() {}
+
+ public static final String[] DATE_PTNS = new String[]{
+ "yyyy-MM-dd", "yyyy/MM/dd", "MM/dd/yyyy", "yyyy.MM.dd",
+ "yyyyMMdd", "yyyyMM", "yyyy-MM", "yyyy/MM"
+ };
+ static final TimestampParser dateParser = new TimestampParser(DATE_PTNS);
+
+ public static java.sql.Date parseDate(String str) {
+ if (str.length() < 6 || str.length() > 10) {
+ throw new IllegalArgumentException("Invalid date value: " + str);
+ }
+ try {
+ Timestamp ts = dateParser.parseTimestamp(str);
+ return new java.sql.Date(ts.getTime());
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Error date value '" + str + "'. "
+ + "Date format must be " + StringUtils.join(DATE_PTNS, " or "), e);
+ }
+ }
+
+ public static final String[] TIMESTAMP_PTNS = new String[]{
+ "yyyy-MM-dd HH:mm",
+ "yyyy-MM-dd HH:mm:ss",
+ "yyyy-MM-dd HH:mm:ss.SSS",
+ "yyyy-MM-dd HH:mm:ss,SSS",
+ "yyyy-MM-dd'T'HH:mm:ss",
+ "yyyy-MM-dd'T'HH:mm:ss.SSS",
+ "yyyy-MM-dd'T'HH:mm:ss'Z'",
+ "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
+ "yyyy/MM/dd HH:mm",
+ "yyyy/MM/dd HH:mm:ss",
+ "MM/dd/yyyy HH:mm",
+ "MM/dd/yyyy HH:mm:ss",
+ };
+ static final TimestampParser timestampParser = new TimestampParser(TIMESTAMP_PTNS);
+
+ public static java.sql.Timestamp parseTimestamp(String str) {
+ if (str.length() < 12 || str.length() > 24) {
+ throw new IllegalArgumentException("Invalid timestamp value: " + str);
+ }
+ try {
+ return timestampParser.parseTimestamp(str);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Error timestamp value '" + str + "'. "
+ + "Timestamp format must be " + StringUtils.join(TIMESTAMP_PTNS, " or "));
+ }
+ }
+
+ public static Boolean parseBoolean(String str) {
+ if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("t") || str.equals("1")) {
+ return true;
+ }
+
+ if (str.equalsIgnoreCase("false") || str.equalsIgnoreCase("f") || str.equals("0")) {
+ return false;
+ }
+
+ throw new IllegalArgumentException("Not a boolean: " + str);
+ }
+
+ /**
+ * Think about a complicated number, like "$100,000.00 %"
+ */
+ private static final int MAX_HEAD_SKIP = 2;
+ private static final int MAX_TAIL_SKIP = 2;
+
+ private static String extractNumberPart(String str, boolean force) {
+ int s = 0;
+ int e = str.length() - 1;
+
+ // find the start of number
+ s = findStartIndexOfNumber(str, force, s, e);
+ // find the end of number
+ e = findEndIndexOfNumber(str, force, s, e);
+
+ if (s > e)
+ throw new NumberFormatException("String '" + str + "' does not contain a number");
+
+ str = str.substring(s, e + 1);
+
+ // filter out any ','
+ for (int i = 0; i < str.length(); i++)
+ if (str.charAt(i) == ',')
+ return str.replace(",", "");
+
+ return str;
+ }
+
+ private static int findEndIndexOfNumber(String str, boolean force, int s, int e) {
+ int tailSkip = 0;
+ for (; s <= e; e--, tailSkip++) {
+ char c = str.charAt(e);
+ if (!force && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || tailSkip > MAX_TAIL_SKIP)) {
+ throw new NumberFormatException("String '" + str + "' does not contain a number");
+ }
+ if (c >= '0' && c <= '9')
+ break;
+ }
+ return e;
+ }
+
+ private static int findStartIndexOfNumber(String str, boolean force, int s, int e) {
+ int headSkip = 0;
+ for (; s <= e; s++, headSkip++) {
+ char c = str.charAt(s);
+ if (!force && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || headSkip > MAX_HEAD_SKIP)) {
+ throw new NumberFormatException("String '" + str + "' does not contain a number");
+ }
+ if (c == '-' || c == '+' || (c >= '0' && c <= '9')) {
+ break;
+ }
+ }
+ return s;
+ }
+
+ public static Double parseDouble(String str) {
+ return parseDouble(str, false);
+ }
+
+ public static Double parseDouble(String str, boolean force) {
+ return Double.parseDouble(extractNumberPart(str, force));
+ }
+
+ public static Float parseFloat(String str) {
+ return parseFloat(str, false);
+ }
+
+ public static Float parseFloat(String str, boolean force) {
+ return Float.parseFloat(extractNumberPart(str, force));
+ }
+
+ public static Integer parseInteger(String str) {
+ return parseInteger(str, false);
+ }
+
+ public static Integer parseInteger(String str, boolean force) {
+ return Integer.parseInt(extractNumberPart(str, force));
+ }
+
+ public static Long parseLong(String str) {
+ return parseLong(str, false);
+ }
+
+ public static Long parseLong(String str, boolean force) {
+ return Long.parseLong(extractNumberPart(str, force));
+ }
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala b/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala
index eb89d4920c..731d4dbf48 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/kylin/softaffinity/SoftAffinityManager.scala
@@ -18,7 +18,7 @@
package org.apache.kylin.softaffinity
-import org.apache.kylin.cache.softaffinity.SoftAffinityConstants
+import org.apache.kylin.cache.softaffinity.{SoftAffinityBookKeeping, SoftAffinityConstants}
import org.apache.kylin.softaffinity.strategy.SoftAffinityStrategy
import java.util.concurrent.locks.ReentrantReadWriteLock
@@ -51,6 +51,11 @@ object SoftAffinityManager extends Logging {
SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED_DEFAULT_VALUE
)
+ lazy val usingSoftAffinityForHivePushdown: Boolean = SparkEnv.get.conf.getBoolean(
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED_FOR_HIVE,
+ SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED_FOR_HIVE_DEFAULT_VALUE
+ )
+
def totalExecutors(): Int = totalRegisteredExecutors.intValue()
def handleExecutorAdded(execHostId: (String, String)): Unit = {
@@ -132,16 +137,39 @@ object SoftAffinityManager extends Logging {
}
}
- def askExecutors(file: String): Array[(String, String)] = {
+ def askExecutors(file: String): Array[String] = {
resourceRWLock.readLock().lock()
try {
if (nodesExecutorsMap.size < 1) {
- Array.empty
- } else {
- softAffinityAllocation.allocateExecs(file, fixedIdForExecutors)
+ return Array.empty
+ }
+
+ val execIdAndHosts = softAffinityAllocation.allocateExecs(file, fixedIdForExecutors)
+
+ val ret = execIdAndHosts.map { p =>
+ if (p._1.equals("")) p._2
+ else s"executor_${p._2}_${p._1}"
+ }
+
+ if (log.isDebugEnabled()) {
+ SoftAffinityBookKeeping.recordAsk(file, ret)
}
+
+ ret
} finally {
resourceRWLock.readLock().unlock()
}
}
+
+ def auditAsks(): java.util.Map[String, String] = {
+ SoftAffinityBookKeeping.audit()
+ }
+
+ def logAuditAsks(): Unit = {
+ if (usingSoftAffinity && log.isDebugEnabled()) {
+ SoftAffinityBookKeeping.logAudits()
+ }
+ }
+
+
}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala b/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala
index cca768ab3d..7e77eaa43a 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CacheFilePartition.scala
@@ -31,10 +31,7 @@ import org.apache.spark.sql.connector.read.InputPartition
case class CacheFilePartition(index: Int, files: Array[CachePartitionedFile])
extends Partition with InputPartition {
override def preferredLocations(): Array[String] = {
- files.head.locations.map { p =>
- if (p._1.equals("")) p._2
- else ExecutorCacheTaskLocation(p._2, p._1).toString
- }
+ files.head.locations
}
}
@@ -45,7 +42,7 @@ object CacheFilePartition extends Logging {
val expectedTargets = filePartition.preferredLocations()
val files = filePartition.files
- var locations = Array.empty[(String, String)]
+ var locations = Array.empty[String]
if (!files.isEmpty && SoftAffinityManager.usingSoftAffinity
&& !SoftAffinityManager.checkTargetHosts(expectedTargets)) {
// if there is no host in the node list which are executors running on,
@@ -56,10 +53,10 @@ object CacheFilePartition extends Logging {
logInfo(s"SAMetrics=File ${files.head.filePath} - " +
s"the expected executors are ${locations.mkString("_")} ")
} else {
- locations = expectedTargets.map(("", _))
+ locations = expectedTargets
}
} else {
- locations = expectedTargets.map(("", _))
+ locations = expectedTargets
}
CacheFilePartition(filePartition.index, filePartition.files.map(p => {
CachePartitionedFile(p.partitionValues, p.filePath, p.start, p.length, locations)
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CachePartitionedFile.scala b/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CachePartitionedFile.scala
index 293975f7f3..cdcc987b94 100644
--- a/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CachePartitionedFile.scala
+++ b/src/spark-project/kylin-soft-affinity-cache/src/main/scala/org/apache/spark/sql/execution/datasources/CachePartitionedFile.scala
@@ -28,7 +28,7 @@ case class CachePartitionedFile(
filePath: String,
start: Long,
length: Long,
- locations: Array[(String, String)] = Array.empty) {
+ locations: Array[String] = Array.empty) {
override def toString: String = {
s"path: $filePath, range: $start-${start + length}, partition values: $partitionValues," +
s" on executor locations ${locations.mkString}"
@@ -41,8 +41,9 @@ class CacheFileScanRDD(
@transient val cacheFilePartitions: Seq[CacheFilePartition])
extends FileScanRDD(sparkSession, readFunction, Nil) {
- def checkCached(cacheLocations: Array[(String, String)]): Boolean = {
- cacheLocations.map(_._1).contains(SparkEnv.get.executorId)
+ def checkCached(cacheLocations: Array[String]): Boolean = {
+ val underscoreExecId = "_" + SparkEnv.get.executorId;
+ cacheLocations.exists(_.contains(underscoreExecId))
}
override def compute(split: RDDPartition, context: TaskContext): Iterator[InternalRow] = {
@@ -50,7 +51,7 @@ class CacheFileScanRDD(
val start = System.currentTimeMillis()
val cacheFilePartition = split.asInstanceOf[CacheFilePartition]
val cacheSplit = FilePartition(cacheFilePartition.index, cacheFilePartition.files.map { f =>
- PartitionedFile(f.partitionValues, f.filePath, f.start, f.length, f.locations.map(_._2))
+ PartitionedFile(f.partitionValues, f.filePath, f.start, f.length, f.locations)
})
var currFilePath = "empty"
val isCache = if (!cacheFilePartition.files.isEmpty) {
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/KylinCacheFileSystemTest.java b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/KylinCacheFileSystemTest.java
deleted file mode 100644
index 321a9d0b99..0000000000
--- a/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/KylinCacheFileSystemTest.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.cache.fs;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.concurrent.ExecutionException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.AbstractTestCase;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.TemporaryFolder;
-
-import org.apache.kylin.cache.kylin.OnlyForTestCacheFileSystem;
-
-public class KylinCacheFileSystemTest extends AbstractTestCase {
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Rule
- public ExpectedException exceptionRule = ExpectedException.none();
-
- public static FileStatus f = null;
-
- @Test
- public void testInitKylinCacheFileSystemWithoutLocalCache() throws Exception {
- Configuration conf = initAndResetConf();
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE, "false");
- OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
- fs.initialize(URI.create("file:/"), conf);
- FileStatus f = createSingleFile("a");
- fs.createNewFile(f.getPath());
- FSDataInputStream stream = fs.open(f.getPath());
- Assert.assertFalse(stream.getWrappedStream() instanceof AlluxioHdfsFileInputStream);
- Assert.assertFalse(stream.getWrappedStream() instanceof CacheFileInputStream);
- }
-
- @Test
- public void testInitKylinCacheFileSystemWithLocalCacheAndUseBufferFile() throws Exception {
- Configuration conf = initAndResetConf();
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE, "true");
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM, "false");
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM, "true");
- OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
- fs.initialize(URI.create("file:/"), conf);
- FileStatus f = createSingleFile("b");
- fs.createNewFile(f.getPath());
- FSDataInputStream stream = fs.open(f.getPath());
- Assert.assertTrue(stream.getWrappedStream() instanceof CacheFileInputStream);
- }
-
- @Test
- public void testInitKylinCacheFileSystemWithLocalCacheWithUseLegacyFile() throws Exception {
- Configuration conf = initAndResetConf();
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE, "true");
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM, "true");
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM, "false");
- OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
- fs.initialize(URI.create("file:/"), conf);
- FileStatus f = createSingleFile("c");
- fs.createNewFile(f.getPath());
- FSDataInputStream stream = fs.open(f.getPath());
- Assert.assertTrue(stream.getWrappedStream() instanceof AlluxioHdfsFileInputStream);
- }
-
- @Test
- public void testFileStatusWithoutCache() throws IOException, ExecutionException {
- Configuration conf = initAndResetConf();
- OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
- fs.initialize(URI.create("file:/"), conf);
- FileStatus f = createSingleFile("d");
- fs.createNewFile(f.getPath());
- Assert.assertEquals(0, fs.fileStatusCache.size());
- }
-
- @Test
- public void testFileStatusWithCache() throws IOException, ExecutionException {
- Configuration conf = initAndResetConf();
- conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_FILE_STATUS_CACHE, "true");
- OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
- fs.initialize(URI.create("file:/"), conf);
- FileStatus f = createSingleFile("d");
- fs.createNewFile(f.getPath());
- fs.getFileStatus(f.getPath());
- Assert.assertEquals(1, fs.fileStatusCache.size());
- }
-
- @Test
- public void testFileStatusCacheWithFileNotFoundException() throws IOException, ExecutionException {
- exceptionRule.expect(ExecutionException.class);
- Configuration conf = initAndResetConf();
- OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
- LocalFileSystem lfs = new LocalFileSystem();
- lfs.initialize(URI.create("file:/"), new Configuration());
- fs.initialize(URI.create("file:/"), conf);
- FileStatus f = createSingleFile("d");
- lfs.createFile(f.getPath());
- fs.fileStatusCache.get(f.getPath());
- }
-
- private Configuration initAndResetConf() {
- Configuration conf = new Configuration();
- conf.set("fs.hdfs.impl", OnlyForTestCacheFileSystem.class.getName());
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
- conf.setBoolean("fs.file.impl.disable.cache", true);
- return conf;
- }
-
- private FileStatus createSingleFile(String fileName) {
- return new FileStatus(1, false, 5, 3, 4, 5, null, "", "", new Path(folder.getRoot().getPath() + fileName));
- }
-}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContentTest.java b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContentTest.java
new file mode 100644
index 0000000000..95da847ba1
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileContentTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import alluxio.client.file.cache.PageId;
+
+public class ManagerOfCacheFileContentTest {
+
+ private ManagerOfCacheFileContent cacheManager;
+
+ @Before
+ public void setUp() {
+ cacheManager = new ManagerOfCacheFileContent(null);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testMarkFilePageRemoved() {
+ // Add some dummy data to the cacheManager before testing
+ PageId pageId1 = new PageId("fileId1", 0);
+ PageId pageId2 = new PageId("fileId1", 1);
+ PageId pageId3 = new PageId("fileId2", 0);
+ PageId pageId4 = new PageId("fileId3", 0);
+
+ cacheManager.markFilePageCached(pageId1);
+ cacheManager.markFilePageCached(pageId2);
+ cacheManager.markFilePageCached(pageId3);
+
+ ConcurrentHashMap<String, ManagerOfCacheFileContent.FileMeta> fileMetas = (ConcurrentHashMap<String, ManagerOfCacheFileContent.FileMeta>) ReflectionTestUtils
+ .getField(cacheManager, "fileMetas");
+ ManagerOfCacheFileContent.FileMeta fileMeta1 = fileMetas.get("fileId1");
+ ManagerOfCacheFileContent.FileMeta fileMeta2 = fileMetas.get("fileId2");
+
+ assertNotNull(fileMeta1);
+ assertNotNull(fileMeta2);
+
+ cacheManager.markFilePageRemoved(pageId1);
+ cacheManager.markFilePageRemoved(pageId2);
+ cacheManager.markFilePageRemoved(pageId4);
+
+ ManagerOfCacheFileContent.FileMeta fileMeta3 = fileMetas.get("fileId3");
+ assertNotNull(fileMeta3);
+
+ assertTrue(fileMeta1.cachedPageIndex.isEmpty());
+ assertFalse(fileMeta2.cachedPageIndex.isEmpty());
+ assertTrue(fileMeta3.cachedPageIndex.isEmpty());
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatusTest.java b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatusTest.java
new file mode 100644
index 0000000000..66913cfc07
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/fs/ManagerOfCacheFileStatusTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.fs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.kylin.guava30.shaded.common.util.concurrent.UncheckedExecutionException;
+import org.junit.Test;
+
+
+public class ManagerOfCacheFileStatusTest {
+
+ @Test
+ public void makeIOException() {
+ {
+ IOException ioException = new IOException("Test IOException");
+ IOException result = ManagerOfCacheFileStatus.makeIOException(ioException);
+ assertSame(ioException, result);
+ }
+
+ {
+ IOException ioException = new IOException("Test IOException");
+ ExecutionException executionException = new ExecutionException(ioException);
+ IOException result = ManagerOfCacheFileStatus.makeIOException(executionException);
+ assertSame(ioException, result);
+ }
+
+ {
+ IOException ioException = new IOException("Test IOException");
+ UncheckedExecutionException uncheckedExecutionException = new UncheckedExecutionException(ioException);
+ IOException result = ManagerOfCacheFileStatus.makeIOException(uncheckedExecutionException);
+ assertSame(ioException, result);
+ }
+
+ {
+ IOException ioException = new IOException("Test IOException");
+ Throwable wrappedException = new RuntimeException(ioException);
+ IOException result = ManagerOfCacheFileStatus.makeIOException(wrappedException);
+ assertSame(ioException, result);
+ }
+
+ {
+ Throwable otherException = new RuntimeException("Test RuntimeException");
+ IOException result = ManagerOfCacheFileStatus.makeIOException(otherException);
+ assertEquals(IOException.class, result.getClass());
+ assertEquals(otherException, result.getCause());
+ }
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/kylin/KylinCacheFileSystemTest.java b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/kylin/KylinCacheFileSystemTest.java
new file mode 100644
index 0000000000..375537465e
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/cache/kylin/KylinCacheFileSystemTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.cache.kylin;
+
+import static org.apache.kylin.cache.kylin.KylinCacheFileSystem.extractAcceptCacheTime;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.AbstractTestCase;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import org.apache.kylin.cache.fs.AbstractCacheFileSystem;
+import org.apache.kylin.cache.fs.AlluxioHdfsFileInputStream;
+import org.apache.kylin.cache.fs.CacheFileInputStream;
+import org.apache.kylin.cache.fs.CacheFileSystemConstants;
+
+public class KylinCacheFileSystemTest extends AbstractTestCase {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @SuppressWarnings("deprecation")
+ @Rule
+ public ExpectedException exceptionRule = ExpectedException.none();
+
+ public static FileStatus f = null;
+
+ @Test
+ public void testExtractAcceptCacheTimeFromSql() {
+ {
+ String[] pair = extractAcceptCacheTime("");
+ Assert.assertArrayEquals(new String[] { "", null }, pair);
+ }
+ {
+ String[] pair = extractAcceptCacheTime("ACCEPT_CACHE_TIME");
+ Assert.assertArrayEquals(new String[] { "ACCEPT_CACHE_TIME", null }, pair);
+ }
+ {
+ String[] pair = extractAcceptCacheTime("select * from xxx /*+ ACCEPT_CACHE_TIME(158176387682000) */");
+ Assert.assertArrayEquals(new String[] { "select * from xxx ", "158176387682000" }, pair);
+ }
+ {
+ String[] pair = extractAcceptCacheTime("/*+ ACCEPT_CACHE_TIME(158176387682000) */\nselect * from xxx");
+ Assert.assertArrayEquals(new String[] { "\nselect * from xxx", "158176387682000" }, pair);
+ }
+ {
+ String[] pair = extractAcceptCacheTime(
+ "/*+ ACCEPT_CACHE_TIME(158176387682000, 2021-03-28T12:33:54) */\nselect * from xxx");
+ Assert.assertArrayEquals(new String[] { "\nselect * from xxx", "158176387682000" }, pair);
+ }
+ // now 2023/2/27: 1677466066216
+ // big enough: 158176387682000
+ System.out.println("System.currentTimeMillis() -- " + System.currentTimeMillis());
+ }
+
+ @Test
+ public void testInitKylinCacheFileSystemWithoutLocalCache() throws Exception {
+ Configuration conf = initAndResetConf();
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE, "false");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("file:/"), conf);
+ FileStatus f = createSingleFile("a");
+ fs.createNewFile(f.getPath());
+ FSDataInputStream stream = fs.open(f.getPath());
+ Assert.assertFalse(stream.getWrappedStream() instanceof AlluxioHdfsFileInputStream);
+ Assert.assertFalse(stream.getWrappedStream() instanceof CacheFileInputStream);
+ }
+ }
+
+ @Test
+ public void testInitKylinCacheFileSystemWithLocalCacheAndUseBufferFile() throws Exception {
+ Configuration conf = initAndResetConf();
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE, "true");
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM, "false");
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM, "true");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("file:/"), conf);
+ FileStatus f = createSingleFile("b");
+ fs.createNewFile(f.getPath());
+ FSDataInputStream stream = fs.open(f.getPath());
+ Assert.assertTrue(stream.getWrappedStream() instanceof CacheFileInputStream);
+ }
+ }
+
+ @Test
+ public void testInitKylinCacheFileSystemWithLocalCacheWithUseLegacyFile() throws Exception {
+ Configuration conf = initAndResetConf();
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_CACHE, "true");
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_LEGACY_FILE_INPUTSTREAM, "true");
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_BUFFER_FILE_INPUTSTREAM, "false");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("file:/"), conf);
+ FileStatus f = createSingleFile("c");
+ fs.createNewFile(f.getPath());
+ FSDataInputStream stream = fs.open(f.getPath());
+ Assert.assertTrue(stream.getWrappedStream() instanceof AlluxioHdfsFileInputStream);
+ }
+ }
+
+ @Test(expected = IOException.class)
+ public void testInitKylinCacheFileSystemWithWrongScheme() throws Exception {
+ Configuration conf = initAndResetConf();
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("noSuchScheme:/"), conf);
+ Assert.fail();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test(expected = IOException.class)
+ public void testInitKylinCacheFileSystemWithSchemaNoClassFound() throws Exception {
+ Configuration conf = initAndResetConf();
+ Map<String, String> schemeClassMap = (Map<String, String>) ReflectionTestUtils
+ .getField(AbstractCacheFileSystem.class, "schemeClassMap");
+ Assert.assertNotNull(schemeClassMap);
+ schemeClassMap.put("noClassFoundScheme", "no.such.class.NoSuchClass");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("noClassFoundScheme:/"), conf);
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testFileStatusWithoutCache() throws IOException {
+ Configuration conf = initAndResetConf();
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_FILE_STATUS_CACHE, "true");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("file:/"), conf);
+ FileStatus f = createSingleFile("d");
+ fs.createNewFile(f.getPath());
+ Assert.assertEquals(0, fs.getStatusCache().size());
+ }
+ }
+
+ @Test
+ public void testFileStatusWithCache() throws IOException {
+ Configuration conf = initAndResetConf();
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_FILE_STATUS_CACHE, "true");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem()) {
+ fs.initialize(URI.create("file:/"), conf);
+ FileStatus f = createSingleFile("d");
+ fs.createNewFile(f.getPath());
+ fs.getFileStatus(f.getPath());
+ Assert.assertEquals(1, fs.getStatusCache().size());
+ }
+ }
+
+ @Test
+ public void testFileStatusCacheWithFileNotFoundException() throws IOException {
+ exceptionRule.expect(FileNotFoundException.class);
+ Configuration conf = initAndResetConf();
+ conf.set(CacheFileSystemConstants.PARAMS_KEY_USE_FILE_STATUS_CACHE, "true");
+ try (OnlyForTestCacheFileSystem fs = new OnlyForTestCacheFileSystem();
+ LocalFileSystem lfs = new LocalFileSystem()) {
+ lfs.initialize(URI.create("file:/"), new Configuration());
+ fs.initialize(URI.create("file:/"), conf);
+ FileStatus f = createSingleFile("d");
+ lfs.createFile(f.getPath());
+ fs.getStatusCache().getFileStatus(f.getPath());
+ }
+ }
+
+ private Configuration initAndResetConf() {
+ Configuration conf = new Configuration();
+ conf.set("fs.hdfs.impl", OnlyForTestCacheFileSystem.class.getName());
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ conf.setBoolean("fs.file.impl.disable.cache", true);
+ return conf;
+ }
+
+ private FileStatus createSingleFile(String fileName) {
+ Path path = new Path(folder.getRoot().getPath() + fileName);
+ return new FileStatus(1, false, 5, 3, 4, 5, null, "", "", path);
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/hive/serde2/lazy/lazy/LazyQuoteAwareSerDeTest.java b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/hive/serde2/lazy/lazy/LazyQuoteAwareSerDeTest.java
new file mode 100644
index 0000000000..d78a9edcc3
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/java/org/apache/kylin/hive/serde2/lazy/lazy/LazyQuoteAwareSerDeTest.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.hive.serde2.lazy.lazy;
+
+import static org.apache.kylin.hive.serde2.lazy.LazyQuoteAwareSerDe.PrimitiveParser.parseDate;
+import static org.apache.kylin.hive.serde2.lazy.LazyQuoteAwareSerDe.PrimitiveParser.parseTimestamp;
+
+import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
+import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
+import org.apache.kylin.hive.serde2.lazy.LazyQuoteAwareSerDe;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LazyQuoteAwareSerDeTest {
+
+ @Test
+ public void testParseCsvLines() {
+ List<String[]> content = LazyQuoteAwareSerDe.parseCsvLines(ImmutableList.of(
+ "Bella1,$5%,2020-07-01,2022-04-14 12:00:00",
+ "Emily1,\"@6,000%\",2021-08-01,2022-04-14 12:00:00",
+ "Coco,10,2012-03-01,2022-04-13 12:00:00"
+ ));
+ String[] expectCol2 = new String[] {"$5%", "@6,000%", "10"};
+ for (int i = 0; i < content.size(); i++) {
+ String[] strs = content.get(i);
+ System.out.println(Arrays.toString(strs));
+ Assert.assertEquals(expectCol2[i], strs[1]);
+ }
+ }
+
+ @Test
+ public void testMissingOrExtraFields() throws Exception {
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ Properties tbl = new Properties();
+ tbl.put("escape.delim", "\\");
+ tbl.put("field.delim", ",");
+ tbl.put("columns", "name,age,birth_day,update_time");
+ tbl.put("columns.types", "string:bigint:date:timestamp");
+ serde.initialize(new Configuration(), tbl);
+
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+
+ {
+ Text text = new Text("\"Coco\",3,2020-11-01,2022-04-13 12:00:00");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Coco", str(struct, 0, soi));
+ }
+ {
+ Text text = new Text("Bella1,5,2020-07-01,2022-04-14 12:00:00,EXTRA_FIELD");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Bella1", str(struct, 0, soi));
+ Assert.assertTrue(serde.hasExtraFieldWarn());
+ Assert.assertFalse(serde.hasMissingFieldWarn());
+ }
+ {
+ Text text = new Text("\"Emily1\",6,2021-08-01");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Emily1", str(struct, 0, soi));
+ Assert.assertEquals("6", str(struct, 1, soi));
+ Assert.assertEquals("2021-08-01", str(struct, 2, soi));
+ Assert.assertNull(str(struct, 3, soi));
+ Assert.assertTrue(serde.hasExtraFieldWarn());
+ Assert.assertTrue(serde.hasMissingFieldWarn());
+ }
+ }
+
+ @Test
+ public void testQuoteEnabled() throws Exception {
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ Properties tbl = new Properties();
+ tbl.put("escape.delim", "\\");
+ tbl.put("field.delim", ",");
+ tbl.put("quote.delim", "\"");
+ tbl.put("columns", "name,age,birth_day,update_time,active");
+ tbl.put("columns.types", "string:bigint:date:timestamp:boolean");
+ serde.initialize(new Configuration(), tbl);
+
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+
+ {
+ Text text = new Text("\"Coco\",\"3\",\"2020-11-01\",\"2022-04-13 12:00:00\",0");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Coco", str(struct, 0, soi));
+ Assert.assertEquals("3", str(struct, 1, soi));
+ Assert.assertEquals("2020-11-01", str(struct, 2, soi));
+ Assert.assertEquals("2022-04-13 12:00:00", str(struct, 3, soi));
+ Assert.assertEquals("false", str(struct, 4, soi));
+ }
+ {
+ Text text = new Text("Bel\"\"la1,$5%,2020/07/01,2022-04-14T12:00:00,true");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Bel\"\"la1", str(struct, 0, soi));
+ Assert.assertEquals("5", str(struct, 1, soi));
+ Assert.assertEquals("2020-07-01", str(struct, 2, soi));
+ Assert.assertEquals("2022-04-14 12:00:00", str(struct, 3, soi));
+ Assert.assertEquals("true", str(struct, 4, soi));
+ }
+ {
+ Text text = new Text("\"Emi\"\"ly1\",\"￥6,000\",8/1/2021,2022-04-14T12:00:00.123Z,F");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Emi\"ly1", str(struct, 0, soi));
+ Assert.assertEquals("6000", str(struct, 1, soi));
+ Assert.assertEquals("2021-08-01", str(struct, 2, soi));
+ Assert.assertEquals("2022-04-14 12:00:00.123", str(struct, 3, soi));
+ Assert.assertEquals("false", str(struct, 4, soi));
+ }
+ }
+
+ @Test
+ public void testBadData() throws Exception {
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ Properties tbl = new Properties();
+ tbl.put("escape.delim", "\\");
+ tbl.put("field.delim", ",");
+ tbl.put("quote.delim", "\"");
+ tbl.put("columns", "name,age,birth_day,update_time");
+ tbl.put("columns.types", "string:bigint:date:timestamp");
+ serde.initialize(new Configuration(), tbl);
+
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+
+ {
+ Text text = new Text("\"Coco\",\"BAD_NUM\",\"BAD_DATE\",\"BAD_TIME\"");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("Coco", str(struct, 0, soi));
+ Assert.assertNull(str(struct, 1, soi));
+ Assert.assertNull(str(struct, 2, soi));
+ Assert.assertNull(str(struct, 3, soi));
+ }
+ {
+ Text text = new Text(",,,");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("", str(struct, 0, soi));
+ Assert.assertNull(str(struct, 1, soi));
+ Assert.assertNull(str(struct, 2, soi));
+ Assert.assertNull(str(struct, 3, soi));
+ }
+ }
+
+ @Test
+ public void testDateTimeParser() {
+ Assert.assertEquals("2022-01-01", parseDate("2022-1-1").toString());
+ Assert.assertEquals("2022-01-01", parseDate("2022/1/1").toString());
+ Assert.assertEquals("2022-01-01", parseDate("1/1/2022").toString());
+ assertException(IllegalArgumentException.class, () -> parseDate(""));
+
+ Assert.assertEquals("2022-01-01 00:00:00.0", parseTimestamp("2022-01-01 00:00:00").toString());
+ Assert.assertEquals("2022-01-01 00:00:00.1", parseTimestamp("2022-01-01 00:00:00.1").toString());
+ Assert.assertEquals("2022-01-01 00:00:00.456", parseTimestamp("2022-01-01 00:00:00,456").toString());
+ Assert.assertEquals("2022-01-01 00:00:00.0", parseTimestamp("2022-01-01T00:00:00").toString());
+ Assert.assertEquals("2022-01-01 00:00:00.12", parseTimestamp("2022-01-01T00:00:00.12").toString());
+ Assert.assertEquals("2022-01-01 00:00:00.0", parseTimestamp("2022-01-01T00:00:00Z").toString());
+ Assert.assertEquals("2022-01-01 00:00:00.123", parseTimestamp("2022-01-01T00:00:00.123Z").toString());
+ assertException(IllegalArgumentException.class, () -> parseTimestamp(""));
+
+ Assert.assertEquals("2022-01-01 13:24:00.0", parseTimestamp("2022-01-01 13:24").toString());
+ }
+
+ @Test
+ public void testNumberForceReadIsFalse() throws Exception {
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ Properties tbl = new Properties();
+ tbl.put("escape.delim", "\\");
+ tbl.put("field.delim", ",");
+ tbl.put("quote.delim", "\"");
+ tbl.put("columns", "int_1,long_2,float_3,double_4");
+ tbl.put("columns.types", "int:bigint:float:double");
+ serde.initialize(new Configuration(), tbl);
+
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+
+ {
+ Text text = new Text("\"¥1\",\"$ 22\",\"1.1%\",\"22.2$$\"");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("1", str(struct, 0, soi));
+ Assert.assertEquals("22", str(struct, 1, soi));
+ Assert.assertEquals("1.1", str(struct, 2, soi));
+ Assert.assertEquals("22.2", str(struct, 3, soi));
+ }
+ {
+ Text text = new Text("\"Q1\",\"$$$22\",\"1.1M\",\"22.2%%%\"");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertNull(str(struct, 0, soi));
+ Assert.assertNull(str(struct, 1, soi));
+ Assert.assertNull(str(struct, 2, soi));
+ Assert.assertNull(str(struct, 3, soi));
+ }
+ }
+
+ @Test
+ public void testNumberForceReadIsTrue() throws Exception {
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ Properties tbl = new Properties();
+ tbl.put("escape.delim", "\\");
+ tbl.put("field.delim", ",");
+ tbl.put("quote.delim", "\"");
+ tbl.put("columns", "int_1,long_2,float_3,double_4");
+ tbl.put("columns.types", "int:bigint:float:double");
+ tbl.put("number.force", "true");
+ serde.initialize(new Configuration(), tbl);
+
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+
+ {
+ Text text = new Text("\"¥1\",\"$ 22\",\"1.1%\",\"22.2$$\"");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("1", str(struct, 0, soi));
+ Assert.assertEquals("22", str(struct, 1, soi));
+ Assert.assertEquals("1.1", str(struct, 2, soi));
+ Assert.assertEquals("22.2", str(struct, 3, soi));
+ }
+ {
+ Text text = new Text("\"Q1\",\"$$$22\",\"1.1M\",\"22.2%%%\"");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("1", str(struct, 0, soi));
+ Assert.assertEquals("22", str(struct, 1, soi));
+ Assert.assertEquals("1.1", str(struct, 2, soi));
+ Assert.assertEquals("22.2", str(struct, 3, soi));
+ }
+ }
+
+ @Test
+ public void testDates() throws Exception {
+ LazyQuoteAwareSerDe serde = new LazyQuoteAwareSerDe();
+ Properties tbl = new Properties();
+ tbl.put("escape.delim", "\\");
+ tbl.put("field.delim", ",");
+ tbl.put("columns", "d1,d2,d3,d4");
+ tbl.put("columns.types", "date,date,date,date");
+ serde.initialize(new Configuration(), tbl);
+
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+
+ {
+ Text text = new Text("20231001,2023/11,2023-01,202301");
+ LazyStruct struct = (LazyStruct) serde.deserialize(text);
+ Assert.assertEquals("2023-10-01", str(struct, 0, soi));
+ Assert.assertEquals("2023-11-01", str(struct, 1, soi));
+ Assert.assertEquals("2023-01-01", str(struct, 2, soi));
+ Assert.assertEquals("2023-01-01", str(struct, 3, soi));
+ }
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private void assertException(Class<?> exClass, Runnable runnable) {
+ try {
+ runnable.run();
+ } catch (Exception ex) {
+ if (exClass != ex.getClass())
+ throw new IllegalStateException("Expect exception type " + exClass + ", but it is: ", ex);
+ return; // good ending
+ }
+ throw new IllegalStateException("Expect exception type " + exClass);
+ }
+
+ private String str(LazyStruct struct, int fieldId, StructObjectInspector soi) {
+ StructField field = soi.getAllStructFieldRefs().get(fieldId);
+ //noinspection unchecked
+ LazyPrimitive<ObjectInspector, Writable> val =
+ (LazyPrimitive<ObjectInspector, Writable>) soi.getStructFieldData(struct, field);
+ return val == null ? null : val.getWritableObject().toString();
+ }
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp.csv b/src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp.csv
new file mode 100644
index 0000000000..d1063d4d07
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp.csv
@@ -0,0 +1,3 @@
+name,age,birth_day,update_time
+Coco,3,2020-11-01,2022-04-13 12:00:00
+Ada,4,2021-11-01,2022-04-13 08:00:00
\ No newline at end of file
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp_data_1.csv b/src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp_data_1.csv
new file mode 100644
index 0000000000..3450eb89c0
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/resources/default_3ywc6z78.emp/emp_data_1.csv
@@ -0,0 +1,4 @@
+name,age,birth_day,update_time
+Bella1,5,2020-07-01,2022-04-14 12:00:00
+Emily1,6,2021-08-01,2022-04-14 12:00:00
+Coco,10,2012-03-01,2022-04-13 12:00:00
\ No newline at end of file
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/resources/log4j2.xml b/src/spark-project/kylin-soft-affinity-cache/src/test/resources/log4j2.xml
new file mode 100644
index 0000000000..b812736439
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/resources/log4j2.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration name="Log4j1">
+ <Appenders>
+ <Console name="stdout" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t] %c{2} : %mask{%m}%n"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Root level="INFO">
+ <AppenderRef ref="stdout"/>
+ </Root>
+ <Logger name="io.kyligence" level="DEBUG"/>
+ <Logger name="org.springframework" level="WARN"/>
+ <Logger name="org.apache.kylin" level="DEBUG"/>
+ <Logger name="io.kyligence.kap.metadata.cube.cuboid.NQueryLayoutChooser" level="INFO"/>
+ <Logger name="org.apache.spark.ui" level="WARN"/>
+ <Logger name="org.apache.spark.sql.execution.ui" level="WARN"/>
+ <Logger name="org.springframework.security" level="INFO"/>
+ <Logger name="org.apache.spark.ContextCleaner" level="WARN"/>
+ <Logger name="io.kyligence.kap.ext" level="INFO"/>
+ <Logger name="io.kyligence.kap.metadata.epoch" level="INFO"/>
+ <Logger name="io.kyligence.kap.common.persistence.transaction" level="INFO"/>
+ </Loggers>
+</Configuration>
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/kylin/cache/kylin/KylinCacheFileSystemWithSparkTest.scala b/src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/kylin/cache/kylin/KylinCacheFileSystemWithSparkTest.scala
new file mode 100644
index 0000000000..00496eff47
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/kylin/cache/kylin/KylinCacheFileSystemWithSparkTest.scala
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.kylin
+
+import org.apache.kylin.cache.fs.CacheFileSystemConstants
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.{SparkFunSuite, TaskContext}
+
+class KylinCacheFileSystemWithSparkTest extends SparkFunSuite with SharedSparkSession {
+
+ protected override def afterEach(): Unit = {
+ super.afterEach()
+ sparkContext.setLocalProperty(CacheFileSystemConstants.PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES, null)
+ sparkContext.setLocalProperty(CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME, null)
+ }
+
+ test("Test use local query for current executor") {
+ val key = CacheFileSystemConstants.PARAMS_KEY_LOCAL_CACHE_FOR_CURRENT_FILES
+
+ // TaskContext.get() == null
+ {
+ val fs: KylinCacheFileSystem = new KylinCacheFileSystem
+ assert(!fs.isUseLocalCacheForCurrentExecutor)
+ }
+
+ // spark.kylin.local-cache.for.current.files is empty
+ {
+ sparkContext.setLocalProperty(key, "")
+ sparkContext.range(0, 1, numSlices = 1).foreachPartition { x =>
+ val ctx = TaskContext.get()
+ assert(ctx.getLocalProperty(key).isEmpty)
+ val fs: KylinCacheFileSystem = new KylinCacheFileSystem
+ assert(fs.isUseLocalCacheForCurrentExecutor)
+ }
+ }
+
+ // spark.kylin.local-cache.for.current.files is "true"
+ {
+ sparkContext.setLocalProperty(key, "true")
+ sparkContext.range(0, 1, numSlices = 1).foreachPartition { x =>
+ val ctx = TaskContext.get()
+ assert(ctx.getLocalProperty(key).eq("true"))
+ val fs: KylinCacheFileSystem = new KylinCacheFileSystem
+ assert(fs.isUseLocalCacheForCurrentExecutor)
+ }
+ }
+ }
+
+ test("Test set accept cache time") {
+ val key = CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME
+
+ // first set
+ {
+ val acceptCacheTime = System.currentTimeMillis()
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(acceptCacheTime)
+ val actual = java.lang.Long.parseLong(sparkContext.getLocalProperty(key))
+ assert(actual == acceptCacheTime)
+ }
+
+ // second set, take the max
+ {
+ val acceptCacheTime = System.currentTimeMillis() + 500L
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(acceptCacheTime)
+ val actual = java.lang.Long.parseLong(sparkContext.getLocalProperty(key))
+ assert(actual == acceptCacheTime)
+ }
+
+ // third set, time range exceeds
+ {
+ val acceptCacheTime = System.currentTimeMillis() + 2000L
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(acceptCacheTime)
+ val actual = java.lang.Long.parseLong(sparkContext.getLocalProperty(key))
+ assert(actual < acceptCacheTime)
+ }
+
+ // clear and set to null, should close to current system time
+ {
+ KylinCacheFileSystem.clearAcceptCacheTimeLocally()
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(null)
+ val actual = java.lang.Long.parseLong(sparkContext.getLocalProperty(key))
+ assert(Math.abs(actual - System.currentTimeMillis()) < 1000)
+ }
+ }
+
+ test("Test clear accept cache time") {
+ val key = CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME
+
+ // Clear accept cache time without TaskContext
+ sparkContext.setLocalProperty(key, "123")
+ KylinCacheFileSystem.clearAcceptCacheTimeLocally()
+ assertResult(null)(sparkContext.getLocalProperty(key))
+ }
+
+ test("Test get accept cache time") {
+ val key = CacheFileSystemConstants.PARAMS_KEY_ACCEPT_CACHE_TIME
+
+ // get accept cache time inside task
+ sparkContext.setLocalProperty(key, "123")
+ sparkContext.range(0, 1, numSlices = 1).foreachPartition { x =>
+ assert(KylinCacheFileSystem.getAcceptCacheTimeLocally == 123L)
+ }
+
+ sparkContext.setLocalProperty(key, null)
+ sparkContext.range(0, 1, numSlices = 1).foreachPartition { x =>
+ val now = System.currentTimeMillis()
+ // time diff 300ms should be enough
+ assert(Math.abs(KylinCacheFileSystem.getAcceptCacheTimeLocally - now) < 300L)
+ }
+
+ // get accept cache time on driver
+ sparkContext.setLocalProperty(key, "456")
+ sparkContext.range(0, 1, numSlices = 1).foreachPartition { x =>
+ assert(KylinCacheFileSystem.getAcceptCacheTimeLocally == 456L)
+ }
+
+ sparkContext.setLocalProperty(key, null)
+ sparkContext.range(0, 1, numSlices = 1).foreachPartition { x =>
+ val now = System.currentTimeMillis()
+ // time diff 300ms should be enough
+ assert(Math.abs(KylinCacheFileSystem.getAcceptCacheTimeLocally - now) < 300L)
+ }
+ }
+
+}
diff --git a/src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/spark/sql/hive/execution/HiveSoftAffinityAndLocalCacheTest.scala b/src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/spark/sql/hive/execution/HiveSoftAffinityAndLocalCacheTest.scala
new file mode 100644
index 0000000000..f1ba173607
--- /dev/null
+++ b/src/spark-project/kylin-soft-affinity-cache/src/test/scala/org/apache/spark/sql/hive/execution/HiveSoftAffinityAndLocalCacheTest.scala
@@ -0,0 +1,213 @@
+/*
+ * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ *
+ * http://kyligence.io
+ *
+ * This software is the confidential and proprietary information of
+ * Kyligence Inc. ("Confidential Information"). You shall not disclose
+ * such Confidential Information and shall use it only in accordance
+ * with the terms of the license agreement you entered into with
+ * Kyligence Inc.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.apache.spark.sql.hive.execution
+
+import org.apache.kylin.cache.softaffinity.SoftAffinityConstants
+import org.apache.kylin.cache.kylin.KylinCacheFileSystem
+import org.apache.kylin.softaffinity.SoftAffinityManager
+import org.apache.kylin.softaffinity.scheduler.SoftAffinityListener
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.mapred.InputSplitWithLocationInfo
+import org.apache.kylin.guava30.shaded.common.io.Resources
+import org.apache.spark.rdd.{HadoopPartition, RDD}
+import org.apache.spark.scheduler.cluster.ExecutorInfo
+import org.apache.spark.scheduler.{SparkListenerExecutorAdded, SparkListenerExecutorRemoved}
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, HiveTableRelation}
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+import org.apache.spark.{Partition, SparkConf}
+
+import java.io.File
+import java.net.URI
+
+// scalastyle:off println
+
+class HiveSoftAffinityAndLocalCacheTest extends SharedSparkSession {
+
+ protected override def beforeEach(): Unit = {
+ super.beforeEach()
+ FileSystem.closeAll()
+ }
+
+ protected override def sparkConf: SparkConf = super.sparkConf
+ .set(SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED, "true")
+ .set(SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_ENABLED_FOR_HIVE, "true")
+ .set(SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_REPLICATIONS_NUM, "2")
+ .set(SoftAffinityConstants.PARAMS_KEY_SOFT_AFFINITY_MIN_TARGET_HOSTS, "2")
+ .set("spark.hadoop.fs.file.impl", "org.apache.kylin.cache.kylin.KylinCacheFileSystem")
+ .set("spark.hadoop.spark.kylin.local-cache.enabled", "true")
+ .set("spark.hadoop.spark.kylin.file-status-cache.enabled", "true")
+ .set("spark.hadoop.spark.kylin.local-cache.use.legacy.file-input-stream", "true")
+ .set("spark.hadoop.alluxio.user.client.cache.store.type", "MEM")
+ .set("spark.hadoop.alluxio.user.client.cache.size", "100MB")
+
+
+
+ test("Test soft affinity") {
+ // ASSERT CONF & INITIAL STATE
+ assert(SoftAffinityManager.usingSoftAffinity)
+ assert(SoftAffinityManager.usingSoftAffinityForHivePushdown)
+ assert(SoftAffinityManager.totalExecutors() == 0)
+
+ // ADD ONE EXECUTOR
+ val executorsListener = new SoftAffinityListener()
+ val addEvent0 = SparkListenerExecutorAdded(System.currentTimeMillis(), "0",
+ new ExecutorInfo("host-1", 3, null))
+ executorsListener.onExecutorAdded(addEvent0)
+ assert(SoftAffinityManager.totalExecutors() == 1)
+
+ // PREP
+ val attrName = AttributeReference("name", StringType)()
+ val attrAge = AttributeReference("age", LongType)()
+ val attrBirthDay = AttributeReference("birth_day", DateType)()
+ val attrUpdateTime = AttributeReference("update_time", TimestampType)()
+ val attrs = Seq(attrName, attrAge, attrBirthDay, attrUpdateTime)
+
+ val storageLoc = CatalogStorageFormat(
+ Option(new URI("file://" +
+ new File(Resources.getResource("default_3ywc6z78.emp").getPath).getAbsolutePath)),
+ Option("org.apache.kylin.cache.softaffinity.SoftAffinityTextInputFormat"), // <---- enable soft affinity for hive pushdown
+ Option("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"),
+ Option("org.apache.kylin.hive.serde2.lazy.LazyQuoteAwareSerDe"), // <---- enable read CSV file with "quote"
+ compressed = false,
+ Map("escape.delim" -> "\\", "field.delim" -> ",")
+ )
+
+ val schema = StructType(Seq(
+ StructField("name", StringType),
+ StructField("age", LongType),
+ StructField("birth_day", DateType),
+ StructField("update_time", TimestampType),
+ ))
+
+ val tableMeta = CatalogTable(
+ TableIdentifier("emp", Option("default_3ywc6z78")),
+ CatalogTableType.EXTERNAL,
+ storageLoc,
+ schema,
+ Option("hive"),
+ properties = Map("classification" -> "csv", "datasource_type" -> "0", "skip.header.line.count" -> "1")
+ )
+
+ val relation = HiveTableRelation(tableMeta, attrs, Seq())
+ val enc = RowEncoder(schema).resolveAndBind()
+ val des = enc.createDeserializer()
+
+ val fs = FileSystem.get(new URI("file:///"), spark.sessionState.newHadoopConf()).asInstanceOf[KylinCacheFileSystem]
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(System.currentTimeMillis())
+
+ // QUERY ROUND 1
+ {
+ // HIVE RDD
+ val exec = HiveTableScanExec(attrs, relation, Seq())(spark)
+ val rdd = exec.execute.map(r => des.apply(r))
+
+ // PRINT PREFERRED LOCATIONS
+ val parts = classOf[RDD[Row]].getDeclaredMethod("getPartitions").invoke(rdd).asInstanceOf[Array[Partition]]
+ for (i <- parts.indices) {
+ val fileSplit = parts(i).asInstanceOf[HadoopPartition].inputSplit.value.asInstanceOf[InputSplitWithLocationInfo]
+ println("partition " + i + " locations: " + fileSplit.getLocationInfo.toStream.map(l => l.getLocation).mkString)
+ }
+
+ // PRINT DATA
+ val dataset = spark.createDataset(rdd)(enc)
+ dataset.show()
+
+ // ASSERT SOFT AFFINITY HAPPENED
+ val records = SoftAffinityManager.auditAsks()
+ assert(records.size() == 1)
+ assert(records.keySet().iterator().next().endsWith("default_3ywc6z78.emp"))
+ assert(records.containsValue("executor_host-1_0"))
+
+ // ASSERT STATE OF CACHE
+ assert(fs.countCachedFilePages() == 2)
+ assert(fs.countCachedFileStatus() == 4)
+ }
+
+ // QUERY ROUND 2, NOTE CACHE SHOULD STAY
+ {
+ val exec = HiveTableScanExec(attrs, relation, Seq())(spark)
+ val rdd = exec.execute.map(r => des.apply(r))
+ val dataset = spark.createDataset(rdd)(enc)
+ dataset.show()
+
+ // ASSERT SOFT AFFINITY HAPPENED
+ val records = SoftAffinityManager.auditAsks()
+ assert(records.size() == 1)
+ assert(records.keySet().iterator().next().endsWith("default_3ywc6z78.emp"))
+ assert(records.containsValue("executor_host-1_0"))
+
+ // ASSERT STATE OF CACHE
+ assert(fs.countCachedFileStatusEvictions() == 0)
+ assert(fs.countCachedFilePages() == 2)
+ assert(fs.countCachedFileStatus() == 4)
+ }
+
+ // CACHE HINT: REFRESH CACHE
+ Thread.sleep(50)
+ KylinCacheFileSystem.setAcceptCacheTimeLocally(System.currentTimeMillis())
+
+ // QUERY ROUND 3, NOTE CACHE EVICTIONS
+ {
+ val exec = HiveTableScanExec(attrs, relation, Seq())(spark)
+ val rdd = exec.execute.map(r => des.apply(r))
+ val dataset = spark.createDataset(rdd)(enc)
+ dataset.show()
+
+ // PRINT SOFT AFFINITY LOGS
+ SoftAffinityManager.logAuditAsks()
+
+ // ASSERT STATE OF CACHE
+ assert(fs.countCachedFileStatusEvictions() == 2)
+ assert(fs.countCachedFilePages() == 2)
+ assert(fs.countCachedFileStatus() == 4)
+ }
+
+ // QUERY ROUND 4, NOTE CACHE SHOULD STAY
+ {
+ val exec = HiveTableScanExec(attrs, relation, Seq())(spark)
+ val rdd = exec.execute.map(r => des.apply(r))
+ val dataset = spark.createDataset(rdd)(enc)
+ dataset.show()
+
+ // PRINT SOFT AFFINITY LOGS
+ SoftAffinityManager.logAuditAsks()
+
+ // ASSERT STATE OF CACHE
+ assert(fs.countCachedFileStatusEvictions() == 2)
+ assert(fs.countCachedFilePages() == 2)
+ assert(fs.countCachedFileStatus() == 4)
+ }
+
+ // CLEANUP
+ val removedEvent0 = SparkListenerExecutorRemoved(System.currentTimeMillis(), "0", "")
+ executorsListener.onExecutorRemoved(removedEvent0)
+ assert(SoftAffinityManager.totalExecutors() == 0)
+ KylinCacheFileSystem.clearAcceptCacheTimeLocally()
+ }
+}
diff --git a/src/spark-project/sparder/pom.xml b/src/spark-project/sparder/pom.xml
index 8751afc17e..0857de4b7c 100644
--- a/src/spark-project/sparder/pom.xml
+++ b/src/spark-project/sparder/pom.xml
@@ -52,6 +52,14 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kap-second-storage-core</artifactId>
</dependency>
+ <!--dependency>
+ <groupId>io.kyligence.ke</groupId>
+ <artifactId>ke-external-catalog-sdk</artifactId>
+ </dependency-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-soft-affinity-cache</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/runtime/MppOnTheFlyProvider.java b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/runtime/MppOnTheFlyProvider.java
new file mode 100644
index 0000000000..1dfa3b0d07
--- /dev/null
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/plugin/runtime/MppOnTheFlyProvider.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ *
+ * http://kyligence.io
+ *
+ * This software is the confidential and proprietary information of
+ * Kyligence Inc. ("Confidential Information"). You shall not disclose
+ * such Confidential Information and shall use it only in accordance
+ * with the terms of the license agreement you entered into with
+ * Kyligence Inc.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.apache.kylin.query.plugin.runtime;
+
+
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+
+import java.util.List;
+
+/**
+ * Provides the ability to compute cuboid layout on-the-fly.
+ */
+public interface MppOnTheFlyProvider {
+
+ /**
+ * Compute layout on-the-fly for segments that don't have it.
+ * <p/>
+ * Return null if all segments have required layout.
+ */
+ LogicalPlan computeMissingLayout(List<NDataSegment> prunedSegments, long layoutId, SparkSession ss);
+}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
index a015f6d2c3..fac5914bf6 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
@@ -23,7 +23,6 @@ import java.sql.Timestamp
import java.util
import java.util.concurrent.{Callable, Executors, TimeUnit, TimeoutException}
import java.util.{UUID, List => JList}
-
import org.apache.commons.lang3.StringUtils
import org.apache.kylin.common.util.{DateFormat, HadoopUtil, Pair}
import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
@@ -42,6 +41,10 @@ import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.sql.{DataFrame, Row, SparderEnv, SparkSession}
import org.slf4j.{Logger, LoggerFactory}
+import org.apache.kylin.cache.kylin.KylinCacheFileSystem
+import org.apache.kylin.softaffinity.SoftAffinityManager
+import org.apache.kylin.fileseg.FileSegments
+
import scala.collection.JavaConverters._
import scala.collection.{immutable, mutable}
import scala.concurrent.duration.Duration
@@ -63,32 +66,47 @@ object SparkSqlClient {
(java.lang.Iterable[JList[String]], Int, JList[StructField]) = {
ss.sparkContext.setLocalProperty("spark.scheduler.pool", "query_pushdown")
HadoopUtil.setCurrentConfiguration(ss.sparkContext.hadoopConfiguration)
- val s = "Start to run sql with SparkSQL..."
val queryId = QueryContext.current().getQueryId
ss.sparkContext.setLocalProperty(QueryToExecutionIDCache.KYLIN_QUERY_ID_KEY, queryId)
- logger.info(s)
+ logger.info("Start to run sql with SparkSQL...")
try {
+ // process cache hint like -- select * from xxx /*+ ACCEPT_CACHE_TIME(158176387682000) */
+ val sqlToRun = KylinCacheFileSystem.processAcceptCacheTimeInSql(sql)
+
val db = if (StringUtils.isNotBlank(project)) {
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv).getDefaultDatabase(project)
} else {
null
}
ss.sessionState.conf.setLocalProperty(DEFAULT_DB, db)
- val df = QueryResultMasks.maskResult(ss.sql(sql))
+ val df = QueryResultMasks.maskResult(ss.sql(sqlToRun))
+ logger.info("SparkSQL returned result DataFrame")
+ QueryContext.current().record("to_spark_plan")
autoSetShufflePartitions(df)
-
- val msg = "SparkSQL returned result DataFrame"
- logger.info(msg)
-
- dfToList(ss, sql, df)
+ QueryContext.current().record("auto_set_parts")
+
+ val iter = if (FileSegments.isSyncFileSegSql(sqlToRun)) {
+ val nParts = df.rdd.getNumPartitions // trigger file scan, whose result will be captured by FileSegmentsDetector
+ (
+ ImmutableList.of(ImmutableList.of(nParts.toString).asInstanceOf[JList[String]]),
+ 1,
+ ImmutableList.of(new StructField("CNT", -5, "BIGINT", 0, 0, false))
+ )
+ } else {
+ dfToList(ss, sqlToRun, df)
+ }
+ QueryContext.current().record("collect_result")
+ SoftAffinityManager.logAuditAsks()
+ iter
} finally {
ss.sessionState.conf.setLocalProperty(DEFAULT_DB, null)
+ KylinCacheFileSystem.clearAcceptCacheTimeLocally()
}
}
- def autoSetShufflePartitions(df: DataFrame): Unit = {
+ private def autoSetShufflePartitions(df: DataFrame): Unit = {
val config = KylinConfig.getInstanceFromEnv
val oriShufflePartition = df.sparkSession.sessionState.conf.getConfString(SHUFFLE_PARTITION).toInt
val isSkip = !config.isAutoSetPushDownPartitions || !ResourceDetectUtils.checkPartitionFilter(df.queryExecution.sparkPlan)
@@ -96,6 +114,15 @@ object SparkSqlClient {
logger.info(s"Skip auto set $SHUFFLE_PARTITION, use origin value $oriShufflePartition")
return
}
+
+ val forced = config.autoSetPushDownPartitionsForced
+ if (forced > 0) {
+ df.sparkSession.sessionState.conf.setLocalProperty(SHUFFLE_PARTITION, forced.toString)
+ QueryContext.current().setShufflePartitions(forced)
+ logger.info(s"Auto force set forced spark.sql.shuffle.partitions $forced")
+ return
+ }
+
val isConcurrency = config.isConcurrencyFetchDataSourceSize
val executor = Executors.newSingleThreadExecutor()
val timeOut = config.getAutoShufflePartitionTimeOut
@@ -152,6 +179,7 @@ object SparkSqlClient {
QueryContext.currentTrace().endLastSpan()
val jobTrace = new SparkJobTrace(jobGroup, QueryContext.currentTrace()
, QueryContext.current().getQueryId, SparderEnv.getSparkSession.sparkContext)
+
val results = df.toIterator()
val resultRows = results._1
val resultSize = results._2
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
index 4a06499092..1cde2767b9 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
@@ -17,9 +17,7 @@
*/
package org.apache.kylin.query.runtime.plan
-import java.util.concurrent.ConcurrentHashMap
-import java.{lang, util}
-
+import org.apache.kylin.common.util.ClassUtil
import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
import org.apache.kylin.engine.spark.utils.{LogEx, LogUtils}
import org.apache.kylin.guava30.shaded.common.base.Joiner
@@ -27,20 +25,25 @@ import org.apache.kylin.guava30.shaded.common.collect.{Lists, Sets}
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate
import org.apache.kylin.metadata.cube.gridtable.NLayoutToGridTableMapping
import org.apache.kylin.metadata.cube.model.{LayoutEntity, NDataSegment, NDataflow}
-import org.apache.kylin.metadata.model.{DeriveInfo, FunctionDesc, NTableMetadataManager, ParameterDesc, TblColRef}
+import org.apache.kylin.metadata.model._
import org.apache.kylin.metadata.realization.HybridRealization
import org.apache.kylin.metadata.tuple.TupleInfo
import org.apache.kylin.query.implicits.sessionToQueryContext
+import org.apache.kylin.query.plugin.runtime.MppOnTheFlyProvider
import org.apache.kylin.query.relnode.{KapRel, OLAPContext}
import org.apache.kylin.query.util.{RuntimeHelper, SparderDerivedUtil}
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.dsl.plans.DslLogicalPlan
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, Union}
import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.manager.SparderLookupManager
-import org.apache.spark.sql.types.{ArrayType, DataTypes, DoubleType, StringType, StructField, StructType}
+import org.apache.spark.sql.types._
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.sql.{Column, DataFrame, Row, SparderEnv, SparkInternalAgent, SparkOperation, SparkSession}
+import java.util.concurrent.ConcurrentHashMap
+import java.{lang, util}
import scala.collection.JavaConverters._
@@ -165,8 +168,10 @@ object TableScanPlan extends LogEx {
val mapping = new NLayoutToGridTableMapping(cuboidLayout)
val columnNames = SchemaProcessor.buildGTSchema(cuboidLayout, mapping, tableName)
- /////////////////////////////////////////////
val kapConfig = KapConfig.wrap(dataflow.getConfig)
+ val mppOnTheFly = getMppOnTheFlyProvider(dataflow.getConfig)
+
+ /////////////////////////////////////////////
val basePath = kapConfig.getReadParquetStoragePath(dataflow.getProject)
if (prunedSegments == null || prunedSegments.size() == 0) {
return tableScanEmptySegment(rel: KapRel)
@@ -195,8 +200,15 @@ object TableScanPlan extends LogEx {
.isFastBitmapEnabled(olapContext.isExactlyFastBitmap)
.bucketingEnabled(bucketEnabled(olapContext, cuboidLayout))
.cuboidTable(dataflow, cuboidLayout, pruningInfo)
- cachePlan.get().put(path, newPlan)
- newPlan
+
+ val mppPlan = mppOnTheFly.computeMissingLayout(prunedSegments, cuboidLayout.getId, session)
+
+ val cuboidAndMppPlan =
+ if (mppPlan == null) newPlan
+ else newPlan.union(mppPlan)
+
+ cachePlan.get().put(path, cuboidAndMppPlan)
+ cuboidAndMppPlan
}
plan = SparkOperation.projectAsAlias(columnNames, plan)
@@ -204,6 +216,25 @@ object TableScanPlan extends LogEx {
SparkOperation.project(schema, newPlan)
}
+ private def getMppOnTheFlyProvider(config: KylinConfig): MppOnTheFlyProvider = {
+ var ret: MppOnTheFlyProvider = null
+ if (config.isMppOnTheFlyLayoutsEnabled) {
+ try {
+ ret = ClassUtil.newInstance(config.getMppOnTheFlyLayoutsProvider).asInstanceOf[MppOnTheFlyProvider]
+ } catch {
+ case e: Exception => logError("failed to instantiate MppOnTheFlyProvider", e)
+ }
+ }
+ if (ret == null)
+ new MppOnTheFlyProvider {
+ override def computeMissingLayout(prunedSegments: util.List[NDataSegment], layoutId: Long, ss: SparkSession): LogicalPlan = {
+ null
+ }
+ }
+ else
+ ret
+ }
+
def bucketEnabled(context: OLAPContext, layout: LayoutEntity): Boolean = {
if (!KylinConfig.getInstanceFromEnv.isShardingJoinOptEnabled) {
return false
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
index 4b1382e6f3..101d5d39f9 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/SparderEnv.scala
@@ -43,7 +43,7 @@ import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasource.{KylinSourceStrategy, LayoutFileSourceStrategy, RewriteInferFiltersFromConstraints}
import org.apache.spark.sql.execution.ui.PostQueryExecutionForKylin
-import org.apache.spark.sql.hive.ReplaceLocationRule
+import org.apache.spark.sql.hive.HiveStorageRule
import org.apache.spark.sql.udf.UdfManager
import org.apache.spark.util.{ThreadUtils, Utils}
import org.apache.spark.{ExecutorAllocationClient, SparkConf, SparkContext}
@@ -226,7 +226,7 @@ object SparderEnv extends Logging {
val sparkSession = isLocalMode match {
case true =>
SparkSession.builder
- .master("local")
+ .master("local[3]")
.appName("sparder-local-sql-context")
.enableHiveSupport()
.getOrCreateKylinSession()
@@ -278,7 +278,7 @@ object SparderEnv extends Logging {
def injectExtensions(sse: SparkSessionExtensions): Unit = {
sse.injectPlannerStrategy(_ => KylinSourceStrategy)
sse.injectPlannerStrategy(_ => LayoutFileSourceStrategy)
- sse.injectPostHocResolutionRule(ReplaceLocationRule)
+ sse.injectPostHocResolutionRule(HiveStorageRule)
sse.injectOptimizerRule(_ => new ConvertInnerJoinToSemiJoin())
if (KapConfig.getInstanceFromEnv.isConstraintPropagationEnabled) {
sse.injectOptimizerRule(_ => RewriteInferFiltersFromConstraints)
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/KylinHiveSessionStateBuilder.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/KylinHiveSessionStateBuilder.scala
index 0f756ed271..d8b40a3731 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/KylinHiveSessionStateBuilder.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/KylinHiveSessionStateBuilder.scala
@@ -17,6 +17,8 @@
*/
package org.apache.spark.sql.hive
+import org.apache.kylin.softaffinity.SoftAffinityManager
+import org.apache.kylin.common.KylinConfig
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
@@ -27,13 +29,10 @@ import org.apache.spark.sql.internal.{BaseSessionStateBuilder, SQLConf, SessionS
/**
* hive session hava some rule exp: find datasource table rule
- *
- * @param sparkSession
- * @param parentState
*/
class KylinHiveSessionStateBuilder(sparkSession: SparkSession,
parentState: Option[SessionState] = None)
- extends HiveSessionStateBuilder(sparkSession, parentState) {
+ extends HiveSessionStateBuilder(sparkSession, parentState) {
private def externalCatalog: HiveExternalCatalog =
session.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog]
@@ -43,30 +42,61 @@ class KylinHiveSessionStateBuilder(sparkSession: SparkSession,
}
-case class ReplaceLocationRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
+object Const {
+ val SoftAffinityInputFormatMap: Map[String, String] = Map(
+ "org.apache.hadoop.mapred.TextInputFormat" -> "io.kyligence.kap.cache.softaffinity.SoftAffinityTextInputFormat"
+ )
+ val SerDeMap: Map[String, String] = Map(
+ "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe" -> "io.kyligence.hive.serde2.lazy.LazyQuoteAwareSerDe"
+ )
+}
+
+case class HiveStorageRule(sparkSession: SparkSession) extends Rule[LogicalPlan] {
+
+ var forceDisableSoftAffinity = false
+
override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
case relation: HiveTableRelation
if DDLUtils.isHiveTable(relation.tableMeta) =>
+
+ // replace file system
val specFS = sparkSession.sessionState.conf.getConf(SQLConf.HIVE_SPECIFIC_FS_LOCATION)
- val specCatalog = FSNamespaceUtils.replaceLocWithSpecPrefix(specFS,
- relation.tableMeta.storage)
- val specTableMeta = relation.tableMeta.copy(storage = specCatalog)
- val specRelation = if (specFS != null && specCatalog.locationUri.isDefined) {
- relation.copy(tableMeta = specTableMeta)
- } else relation
+ var specStorage = FSNamespaceUtils.replaceLocWithSpecPrefix(specFS, relation.tableMeta.storage)
+
+ // replace input format, for soft affinity during hive pushdown
+ val softAffinityEnabled = !forceDisableSoftAffinity && SoftAffinityManager.usingSoftAffinity && SoftAffinityManager.usingSoftAffinityForHivePushdown
+ val oldInputFormat = relation.tableMeta.storage.inputFormat
+ val newInputFormat = if (oldInputFormat.isEmpty || !softAffinityEnabled) {
+ oldInputFormat
+ } else {
+ Option(Const.SoftAffinityInputFormatMap.getOrElse(oldInputFormat.get, oldInputFormat.get))
+ }
+
+ // replace serde, for CSV quote support
+ val oldSerDe = relation.tableMeta.storage.serde
+ val newSerDe = if (oldSerDe.isEmpty || !KylinConfig.getInstanceFromEnv.isSupportPushdownHiveCsvEnhancement) {
+ oldSerDe
+ } else {
+ Option(Const.SerDeMap.getOrElse(oldSerDe.get, oldSerDe.get))
+ }
+
+ specStorage = specStorage.copy(inputFormat = newInputFormat, serde = newSerDe)
+ val specTableMeta = relation.tableMeta.copy(storage = specStorage)
+ val specRelation = relation.copy(tableMeta = specTableMeta)
specRelation
}
+
+ def setForceDisableSoftAffinity(value: Boolean): Unit = {
+ forceDisableSoftAffinity = value
+ }
}
/**
* use for no hive mode
- *
- * @param sparkSession
- * @param parentState
*/
class KylinSessionStateBuilder(sparkSession: SparkSession,
parentState: Option[SessionState] = None)
- extends BaseSessionStateBuilder(sparkSession, parentState) {
+ extends BaseSessionStateBuilder(sparkSession, parentState) {
override protected def newBuilder: NewBuilder =
new KylinSessionStateBuilder(_, _)
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/hive/ReplaceLocationRuleTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/hive/ReplaceLocationRuleTest.scala
index 991e427cbd..11dee578b4 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/hive/ReplaceLocationRuleTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/hive/ReplaceLocationRuleTest.scala
@@ -43,11 +43,11 @@ class ReplaceLocationRuleTest extends SparderBaseFunSuite with SharedSparkSessio
)
val relation = new HiveTableRelation(table, table.dataSchema.asNullable.toAttributes, table.partitionSchema.asNullable.toAttributes)
- val afterRulePlan = ReplaceLocationRule.apply(spark).apply(relation)
+ val afterRulePlan = HiveStorageRule.apply(spark).apply(relation)
assert(afterRulePlan.asInstanceOf[HiveTableRelation].tableMeta.storage.toString() == "Storage(Location: hdfs://hacluster)")
spark.sessionState.conf.setLocalProperty("spark.sql.hive.specific.fs.location", "hdfs://writecluster")
- val afterRulePlan1 = ReplaceLocationRule.apply(spark).apply(relation)
+ val afterRulePlan1 = HiveStorageRule.apply(spark).apply(relation)
assert(afterRulePlan1.asInstanceOf[HiveTableRelation].tableMeta.storage.toString() == "Storage(Location: hdfs://writecluster)")
}
@@ -64,7 +64,7 @@ class ReplaceLocationRuleTest extends SparderBaseFunSuite with SharedSparkSessio
)
val relation = new HiveTableRelation(table, table.dataSchema.asNullable.toAttributes, table.partitionSchema.asNullable.toAttributes)
- val afterRulePlan = ReplaceLocationRule.apply(spark).apply(relation)
+ val afterRulePlan = HiveStorageRule.apply(spark).apply(relation)
assert(afterRulePlan.asInstanceOf[HiveTableRelation].tableMeta.storage.toString() == "Storage(Location: hdfs://hacluster)")
val table1 = CatalogTable(
@@ -78,11 +78,11 @@ class ReplaceLocationRuleTest extends SparderBaseFunSuite with SharedSparkSessio
)
val relation1 = new HiveTableRelation(table1, table1.dataSchema.asNullable.toAttributes, table1.partitionSchema.asNullable.toAttributes)
- val afterRulePlan1 = ReplaceLocationRule.apply(spark).apply(relation1)
+ val afterRulePlan1 = HiveStorageRule.apply(spark).apply(relation1)
assert(afterRulePlan1.asInstanceOf[HiveTableRelation].tableMeta.storage.toString() == "Storage()")
try {
- ReplaceLocationRule.apply(spark).apply(null)
+ HiveStorageRule.apply(spark).apply(null)
} catch {
case _: Exception =>
}
@@ -90,7 +90,7 @@ class ReplaceLocationRuleTest extends SparderBaseFunSuite with SharedSparkSessio
val localRelation = LocalRelation(
Seq(AttributeReference("a", IntegerType, nullable = true)()), isStreaming = false)
try {
- ReplaceLocationRule.apply(spark).apply(localRelation)
+ HiveStorageRule.apply(spark).apply(localRelation)
} catch {
case _: Exception =>
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
index aaf33b3e4b..c68b727259 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
@@ -265,6 +265,10 @@ class FilePruner(val session: SparkSession,
logInfo(s"Segment Num: Before filter: ${prunedSegmentDirs.size}, After time partition filter: " +
s"$filteredSizeAfterTimePartition, After dimension filter: ${filteredSizeAfterDimensionFilter}, " +
s"After derived dimension filter: ${selected.size}.")
+
+ selected = selected.filter( // tolerate for ready & empty segment
+ seg => dataflow.getSegment(seg.segmentID).getLayoutsMap.containsKey(layout.getId))
+
selected = selected.par.map { e =>
val logString = s"[fetch file status for Segment ID: ${e.segmentID}; Partition Num: ${e.partitions.size}]"
logTime(logString, true) {
(kylin) 19/35: KYLIN-5815 Failed to add primaryKey of table when upgrading concurrently
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 768a5ececcd855b0f796881d42b36b17088c61d5
Author: Jiawei Li <10...@qq.com>
AuthorDate: Thu Sep 21 14:20:24 2023 +0800
KYLIN-5815 Failed to add primaryKey of table when upgrading concurrently
---
.../common/persistence/metadata/JdbcEpochStore.java | 20 ++++++++++++--------
.../common/persistence/metadata/jdbc/JdbcUtil.java | 10 ++++++++++
.../persistence/metadata/jdbc/JdbcUtilTest.java | 17 +++++++++++++++++
3 files changed, 39 insertions(+), 8 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
index 97b50d2969..2d510801b6 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
@@ -41,6 +41,7 @@ import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
@@ -122,17 +123,20 @@ public class JdbcEpochStore extends EpochStore {
}
return dataSource.getConnection();
}
-
+
@Override
public void createIfNotExist() throws Exception {
if (isTableExists(getConnection(jdbcTemplate), table)) {
- if (!isPrimaryKeyExists(getConnection(jdbcTemplate), table)) {
- withTransaction(transactionManager, () -> {
- jdbcTemplate.execute(getAddPrimarykeySql(table));
- return 1;
- });
- log.info("Succeed to add table primary key: {}", table);
- }
+ JdbcUtil.retry(() -> {
+ if (!isPrimaryKeyExists(getConnection(jdbcTemplate), table)) {
+ withTransaction(transactionManager, () -> {
+ jdbcTemplate.execute(getAddPrimarykeySql(table));
+ return 1;
+ });
+ log.info("Succeed to add table primary key: {}", table);
+ }
+ return 1;
+ });
return;
}
String fileName = "metadata-jdbc-default.properties";
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
index 51b61c348a..265777c1a3 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
@@ -260,4 +260,14 @@ public class JdbcUtil {
val dataSource = BasicDataSourceFactory.createDataSource(props);
return new JdbcTemplate(dataSource);
}
+
+ public static <T> T retry(Callback<T> func) throws Exception {
+ try {
+ return func.handle();
+ } catch (Exception e) {
+ log.error("occur exception, retry", e);
+ return func.handle();
+ }
+ }
+
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
index ab0f7926c9..a3d05f453d 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
@@ -20,6 +20,8 @@ package org.apache.kylin.common.persistence.metadata.jdbc;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.util.HashSet;
+import java.util.Set;
import org.junit.Assert;
import org.junit.Before;
@@ -51,4 +53,19 @@ public class JdbcUtilTest {
null);
Assert.assertFalse(JdbcUtil.isColumnExists(connection, table, "not_exists"));
}
+
+ @Test
+ public void testRetry() throws Exception {
+ Set set = new HashSet();
+ int result = JdbcUtil.retry(() -> {
+ boolean shouldThrowException = set.isEmpty();
+ set.add(true);
+ if (shouldThrowException) {
+ throw new Exception("test");
+ }
+ return 1;
+ });
+ Assert.assertEquals(1, result);
+
+ }
}
(kylin) 15/35: KYLIN-5809 Add TableIndex suggestion control in open api
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 630a3aa7cef2e2c36653d2d92915b01c201d1ae6
Author: lixiang <44...@qq.com>
AuthorDate: Sat Sep 2 11:24:20 2023 +0800
KYLIN-5809 Add TableIndex suggestion control in open api
---
.../java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
index 8ab786b427..7b8959e374 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
@@ -43,6 +43,8 @@ public class LayoutRecDetailResponse implements Serializable {
private List<RecMeasure> measures = Lists.newArrayList();
@JsonProperty("computed_columns")
private List<RecComputedColumn> computedColumns = Lists.newArrayList();
+ @JsonProperty("discarded")
+ private boolean discarded;
@Getter
@Setter
(kylin) 27/35: KYLIN-5836 Fix mapr env building error
Posted by li...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e49c2f15a645cb33a0f04a0b894af8be24613bf5
Author: Yinghao Lin <yi...@kyligence.io>
AuthorDate: Mon Sep 11 11:52:48 2023 +0800
KYLIN-5836 Fix mapr env building error
---
.../org/apache/kylin/engine/spark/application/SparkApplication.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index 1f1d359eb6..2c2dc08bed 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -294,7 +294,7 @@ public abstract class SparkApplication implements Application {
monitorSparkMaster();
- HadoopUtil.setCurrentConfiguration(HadoopUtil.getHadoopConfFromSparkEngine());
+ HadoopUtil.setCurrentConfiguration(HadoopUtil.getCurrentConfiguration());
////////
exchangeSparkConf(buildEnv.sparkConf());