You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2024/04/12 08:24:15 UTC
(kylin) 12/35: KYLIN-5805 Refactor project lock
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 085d7072167bf199094833ef73f0d9cc5741b075
Author: Zhiting Guo <35...@users.noreply.github.com>
AuthorDate: Fri Sep 1 15:59:08 2023 +0800
KYLIN-5805 Refactor project lock
---------
Co-authored-by: junqing.cai <ju...@kyligence.io>
Co-authored-by: Jiale He <ji...@gmail.com>
Co-authored-by: Zhiting Guo <zh...@kyligence.io>
---
build/bin/create-table.sh | 33 +++
build/conf/kylin-tools-log4j.xml | 6 +
.../kap/secondstorage/metadata/Manager.java | 23 +-
.../kap/secondstorage/metadata/NodeGroup.java | 5 +
.../kap/secondstorage/metadata/TablePlan.java | 5 +
.../kylin/rest/controller/NSystemController.java | 33 +++
.../apache/kylin/rest/config/AppInitializer.java | 2 +
.../config/initialize/ModelBrokenListener.java | 12 +-
.../kylin/rest/service/KylinUserService.java | 8 +-
.../apache/kylin/rest/service/ProjectService.java | 5 +-
.../apache/kylin/rest/service/SystemService.java | 44 +++-
.../service/task/QueryHistoryTaskScheduler.java | 21 +-
.../task/RecommendationTopNUpdateScheduler.java | 16 +-
.../task/QueryHistoryTaskSchedulerTest.java | 17 +-
.../rest/util/InitResourceGroupUtilsTest.java} | 34 ++-
src/core-common/pom.xml | 9 +
.../org/apache/kylin/common/KylinConfigBase.java | 25 ++
.../common/exception/code/ErrorCodeSystem.java | 18 +-
.../common/persistence/InMemResourceStore.java | 7 +-
.../kylin/common/persistence/ResourceStore.java | 25 +-
.../common/persistence/RootPersistentEntity.java | 16 +-
.../persistence/ThreadViewResourceStore.java | 13 +-
.../persistence/lock/AbstractMemoryLock.java | 62 +++++
.../persistence/lock/DeadLockException.java} | 26 +--
.../common/persistence/lock/DeadLockHandler.java | 61 +++++
.../common/persistence/lock/DeadLockInfo.java | 72 ++++++
.../common/persistence/lock/IntentionLock.java} | 26 +--
.../persistence/lock/IntentionReadWriteLock.java | 257 +++++++++++++++++++++
.../persistence/lock/IntentionShareLock.java | 59 +++++
.../persistence/lock/LockInterruptException.java} | 26 +--
.../common/persistence/lock/LockManager.java} | 36 ++-
.../persistence/lock/LockManagerInstance.java} | 27 +--
.../persistence/lock/LockTimeoutException.java} | 26 +--
.../kylin/common/persistence/lock/MemoryLock.java} | 30 +--
.../common/persistence/lock/MemoryLockGraph.java | 171 ++++++++++++++
.../common/persistence/lock/MemoryLockManager.java | 97 ++++++++
.../common/persistence/lock/MemoryLockUtils.java | 206 +++++++++++++++++
.../kylin/common/persistence/lock/ModuleLock.java} | 42 ++--
.../common/persistence/lock/ModuleLockEnum.java | 87 +++++++
.../lock/OptimisticLockRetryException.java} | 25 +-
.../kylin/common/persistence/lock/PathLock.java} | 35 +--
.../common/persistence/lock/ProjectLock.java} | 36 +--
.../persistence/lock/ResourcePathParser.java | 121 ++++++++++
.../kylin/common/persistence/lock/TempLock.java} | 25 +-
.../lock/TransactionDeadLockHandler.java | 168 ++++++++++++++
.../common/persistence/lock/TransactionLock.java} | 29 +--
.../persistence/lock/TransactionLockManager.java} | 34 ++-
.../persistence/lock/TransactionProjectLock.java} | 26 +--
.../persistence/lock/TransactionTempLock.java} | 27 +--
.../lock/rule/LockEliminationRule.java} | 33 ++-
.../lock/rule/LockEliminationRuleExec.java | 57 +++++
.../common/persistence/lock/rule/LockInfo.java | 49 ++++
.../lock/rule/ModuleEliminationRule.java | 55 +++++
.../persistence/lock/rule/PathEliminationRule.java | 70 ++++++
.../lock/rule/ProjectEliminationRule.java} | 36 ++-
.../persistence/metadata/JdbcMetadataStore.java | 3 +
.../common/persistence/metadata/MetadataStore.java | 1 +
.../transaction/MessageSynchronization.java | 7 +-
.../persistence/transaction/TransactionLock.java | 96 --------
.../transaction/TransactionManagerInstance.java} | 32 ++-
.../TransactionOptimisticLockManager.java} | 26 +--
.../TransactionPessimisticLockManager.java | 81 +++++++
.../common/persistence/transaction/UnitOfWork.java | 163 +++++++++----
.../persistence/transaction/UnitOfWorkContext.java | 28 ++-
.../persistence/transaction/UnitOfWorkParams.java | 24 ++
.../org/apache/kylin/common/util/JsonUtil.java | 8 +
.../resources/kylin_error_msg_conf_cn.properties | 3 +
.../resources/kylin_error_msg_conf_en.properties | 3 +
.../main/resources/kylin_errorcode_conf.properties | 3 +
.../persistence/JdbcAuditLogRecoveryTest.java | 24 +-
.../common/persistence/JdbcMetadataStoreTest.java | 45 +++-
.../kylin/common/persistence/UnitOfWorkTest.java | 113 ++++++---
.../common/persistence/lock/CloseWaitTest.java | 167 +++++++++++++
.../persistence/lock/MemoryLockGraphTest.java | 218 +++++++++++++++++
.../lock/rule/LockEliminationRuleExecTest.java | 159 +++++++++++++
.../metadata/JdbcAuditLogStoreTest.java | 8 +-
...ckTest.java => TransactionProjectLockTest.java} | 13 +-
.../org/apache/kylin/job/common/SegmentUtil.java | 5 +-
.../org/apache/kylin/job/dao/JobStatistics.java | 10 +-
.../apache/kylin/job/dao/JobStatisticsBasic.java | 12 +-
.../apache/kylin/job/dao/JobStatisticsManager.java | 28 ++-
.../org/apache/kylin/job/dao/NExecutableDao.java | 20 +-
.../kylin/job/execution/NExecutableManager.java | 13 +-
.../kylin/metadata/TempStatementManager.java | 10 +-
.../java/org/apache/kylin/metadata/acl/AclTCR.java | 17 +-
.../apache/kylin/metadata/acl/AclTCRManager.java | 21 +-
.../metadata/cachesync/CacheReloadChecker.java | 6 +-
.../kylin/metadata/cachesync/CachedCrudAssist.java | 53 ++++-
.../kylin/metadata/cube/model/IndexPlan.java | 8 +
.../cube/model/NDataLoadingRangeManager.java | 111 +++++----
.../cube/model/NDataSegDetailsManager.java | 41 +++-
.../metadata/cube/model/NDataflowManager.java | 99 ++++----
.../metadata/cube/model/NIndexPlanManager.java | 38 +--
.../kylin/metadata/favorite/AbstractAsyncTask.java | 11 +
.../metadata/favorite/AsyncAccelerationTask.java | 3 +-
.../kylin/metadata/favorite/AsyncTaskManager.java | 30 ++-
.../kylin/metadata/favorite/FavoriteRule.java | 17 +-
.../metadata/favorite/FavoriteRuleManager.java | 22 +-
.../metadata/favorite/QueryHistoryIdOffset.java | 11 +
.../favorite/QueryHistoryIdOffsetManager.java | 25 +-
.../apache/kylin/metadata/jar/JarInfoManager.java | 34 ++-
.../kylin/metadata/model/ComputedColumnDesc.java | 4 +
.../apache/kylin/metadata/model/FusionModel.java | 10 +-
.../kylin/metadata/model/FusionModelManager.java | 11 +-
.../apache/kylin/metadata/model/NDataModel.java | 24 ++
.../kylin/metadata/model/NDataModelManager.java | 31 ++-
.../metadata/model/NTableMetadataManager.java | 115 ++++++---
.../org/apache/kylin/metadata/model/TableDesc.java | 10 +-
.../apache/kylin/metadata/model/TableExtDesc.java | 19 +-
.../metadata/model/schema/ImportModelContext.java | 2 +-
.../kylin/metadata/project/NProjectManager.java | 12 +-
.../resourcegroup/ResourceGroupManager.java | 2 +-
.../metadata/sourceusage/SourceUsageManager.java | 9 +-
.../metadata/streaming/DataParserManager.java | 20 +-
.../metadata/streaming/KafkaConfigManager.java | 17 +-
.../kylin/metadata/streaming/StreamingManager.java | 13 +-
.../metadata/upgrade/GlobalAclVersionManager.java | 10 +-
.../apache/kylin/metadata/user/ManagedUser.java | 7 +
.../kylin/metadata/user/NKylinUserManager.java | 37 ++-
.../metadata/usergroup/NUserGroupManager.java | 4 +-
.../apache/kylin/metadata/usergroup/UserGroup.java | 7 +
.../org/apache/kylin/rest/security/AclManager.java | 18 +-
.../org/apache/kylin/rest/security/AclRecord.java | 2 +-
.../org/apache/kylin/rest/security/UserAcl.java | 7 +
.../apache/kylin/rest/security/UserAclManager.java | 4 +-
.../org/apache/kylin/util/BrokenEntityProxy.java | 10 +-
.../metadata/cachesync/CachedCrudAssistTest.java | 63 +++++
.../apache/kylin/metadata/cube/IndexPlanTest.java | 49 ++--
.../cube/model/NDataLoadingRangeManagerTest.java | 28 +--
.../cube/model/NDataSegDetailsManagerTest.java | 2 +-
.../metadata/epoch/EnhancedUnitOfWorkTest.java | 8 +-
.../metadata/model/NDataModelManagerTest.java | 50 ++++
.../metadata/project/NProjectManagerTest.java | 32 +++
.../org/apache/kylin/model/NDataModelTest.java | 53 +++++
.../sdk/datasource/PushdownRunnerSDKImplTest.java | 10 +-
.../apache/kylin/rest/service/TableExtService.java | 2 +-
.../apache/kylin/rest/service/TableService.java | 114 +++++----
.../rest/service/StreamingTableServiceTest.java | 4 -
.../e78a89dd-847f-4574-8afa-8768b4228b72.json | 1 -
.../e78a89dd-847f-4574-8afa-8768b4228b73.json | 1 -
.../e78a89dd-847f-4574-8afa-8768b4228b74.json | 1 -
.../org/apache/kylin/newten/AutoMergeTest.java | 7 +-
.../org/apache/kylin/event/HAMetadataTest.java | 23 +-
.../kylin/rest/controller/NTableController.java | 5 +-
.../apache/kylin/rest/request/ModelRequest.java | 4 +
.../apache/kylin/rest/service/ModelService.java | 160 ++++++-------
.../kylin/rest/service/IndexPlanServiceTest.java | 13 +-
.../kylin/rest/service/ModelServiceTest.java | 3 +-
.../kylin/rest/service/TableReloadServiceTest.java | 8 +-
.../kylin/rest/service/TableServiceTest.java | 4 +-
.../apache/kylin/rest/service/QueryService.java | 23 +-
.../kylin/rest/service/QueryServiceTest.java | 22 ++
.../rest/util/QueryCacheSignatureUtilTest.java | 16 +-
.../apache/kylin/query/blacklist/SQLBlacklist.java | 13 +-
.../kylin/query/blacklist/SQLBlacklistManager.java | 28 ++-
.../rest/health/MetaStoreHealthIndicator.java | 3 +-
.../kylin/engine/spark/job/NSparkExecutable.java | 28 ++-
.../engine/spark/job/NSparkUpdateMetadataStep.java | 6 +-
.../spark/builder/SnapshotPartitionBuilder.scala | 6 +-
.../kylin/engine/spark/job/JobManagerTest.java | 5 +-
.../query/pushdown/PushDownRunnerJdbcImplTest.java | 56 ++---
.../streaming/manager/StreamingJobManager.java | 9 +-
.../apache/kylin/helper/MetadataToolHelper.java | 21 +-
.../apache/kylin/tool/garbage/StorageCleaner.java | 22 +-
.../kylin/tool/security/KylinPasswordResetCLI.java | 10 +-
.../kylin/tool/upgrade/UpdateProjectCLI.java | 29 +--
.../util/ProjectTemporaryTableCleanerHelper.java | 4 +-
.../org/apache/kylin/tool/AuditLogToolTest.java | 11 +-
.../apache/kylin/tool/MetadataToolTestFixture.java | 2 +-
.../apache/kylin/tool/YarnApplicationToolTest.java | 11 +-
.../tool/security/KylinPasswordResetCLITest.java | 11 +-
.../ProjectTemporaryTableCleanerHelperTest.java | 5 +-
172 files changed, 4427 insertions(+), 1348 deletions(-)
diff --git a/build/bin/create-table.sh b/build/bin/create-table.sh
new file mode 100755
index 0000000000..5fbc49541d
--- /dev/null
+++ b/build/bin/create-table.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z $KYLIN_HOME ];then
+ export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
+fi
+
+function help {
+ echo "usage: create-table.sh table_path (the table path like /metadata_backup_path/project_name/table/)"
+ exit 1
+}
+
+if [ $# -ne 1 ]; then
+ help
+else
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.util.CreateTableFromJson $1
+fi
diff --git a/build/conf/kylin-tools-log4j.xml b/build/conf/kylin-tools-log4j.xml
index e5e40d0beb..2abe924057 100644
--- a/build/conf/kylin-tools-log4j.xml
+++ b/build/conf/kylin-tools-log4j.xml
@@ -20,6 +20,9 @@
<Console name="stderr" target="SYSTEM_ERR">
<PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p %X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
+ <File name="createTableSql" fileName="${env:KYLIN_HOME}/logs/create_table.sql">
+ <PatternLayout pattern="%msg%n"/>
+ </File>
</Appenders>
<Loggers>
<Root level="INFO">
@@ -32,5 +35,8 @@
<Logger name="io.kyligence" level="DEBUG"/>
<Logger name="org.apache.kylin.tool.shaded" level="INFO"/>
<Logger name="org.apache.kylin" level="DEBUG"/>
+ <Logger name="org.apache.kylin.tool.util.CreateTableFromJson" level="INFO">
+ <AppenderRef ref="createTableSql"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
index 83a5d75211..5e14ee75e7 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
@@ -28,11 +28,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
// CALL FROM CORE
@NotThreadSafe
@@ -69,6 +67,10 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
this.crud.setCheckCopyOnWrite(true);
}
+ public String getResourcePath(String resourceName) {
+ return crud.resourcePath(resourceName);
+ }
+
@SuppressWarnings("unchecked")
protected T save(T entity) {
if (entity instanceof IManagerAware) {
@@ -81,6 +83,10 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
protected T copy(T entity) {
return crud.copyBySerialization(entity);
}
+
+ protected T copyForWrite(T entity) {
+ return crud.copyForWrite(entity);
+ }
// Create
protected abstract T newRootEntity(String cubeName);
@@ -92,10 +98,11 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
public T createAS(T entity) {
if (entity.getUuid() == null)
throw new IllegalArgumentException();
- if (crud.contains(entity.getUuid()))
+ T copy = copyForWrite(entity);
+ if (crud.contains(copy.getUuid()))
throw new IllegalArgumentException("Entity '" + entity.getUuid() + "' already exists");
// overwrite point
- return save(entity);
+ return save(copy);
}
// Read
@@ -126,14 +133,14 @@ public abstract class Manager<T extends RootPersistentEntity> implements IManage
}
public T update(String uuid, Consumer<T> updater) {
- return get(uuid).map(this::copy).map(copied -> {
+ return get(uuid).map(this::copyForWrite).map(copied -> {
updater.accept(copied);
return internalUpdate(copied);
}).orElse(null);
}
protected T upsert(String uuid, Consumer<T> updater, Supplier<T> creator) {
- return get(uuid).map(this::copy).map(copied -> {
+ return get(uuid).map(this::copyForWrite).map(copied -> {
updater.accept(copied);
return internalUpdate(copied);
}).orElseGet(() -> {
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
index b915326ba1..d328d43104 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
@@ -81,6 +81,11 @@ public class NodeGroup extends RootPersistentEntity implements Serializable,
return this;
}
+ @Override
+ public String getResourcePath() {
+ return manager.getResourcePath(resourceName());
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
index ea1fe04af2..c6eb1838a6 100644
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
+++ b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
@@ -145,6 +145,11 @@ public class TablePlan extends RootPersistentEntity
return description;
}
+ @Override
+ public String getResourcePath() {
+ return manager.getResourcePath(resourceName());
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
index b29d4a4eaf..50fe881663 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
@@ -26,6 +26,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.TIME_INVALI
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -38,6 +39,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.persistence.lock.DeadLockInfo;
import org.apache.kylin.common.persistence.transaction.EpochCheckBroadcastNotifier;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
@@ -390,4 +392,35 @@ public class NSystemController extends NBasicController {
setDownloadResponse(backupInputStream, METADATA_FILE, MediaType.APPLICATION_OCTET_STREAM_VALUE, response);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, "", "");
}
+
+ @PostMapping(value = "/deadlock/detect")
+ @ResponseBody
+ public EnvelopeResponse<List<DeadLockInfo>> detectDeadLock() {
+ return new EnvelopeResponse<>(CODE_SUCCESS, systemService.detectDeadLock(), "");
+ }
+
+ @PostMapping(value = "/deadlock/kill")
+ @ResponseBody
+ public EnvelopeResponse<String> killThread(@RequestParam(value = "ids") List<Long> ids) {
+ checkCollectionRequiredArg("ids", ids);
+ systemService.killDeadLockThread(ids);
+ return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
+ }
+
+ @PostMapping(value = "/deadlock/kill_all")
+ @ResponseBody
+ public EnvelopeResponse<String> killAllThread() {
+ systemService.killAllDeadLockThread();
+ return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
+ }
+
+ /**
+ * RPC Call
+ */
+ @DeleteMapping(value = "clean_sparder_event_log")
+ @ResponseBody
+ public EnvelopeResponse<String> queryNodeCleanSparderEventsLogs() {
+ systemService.cleanSparderEventLog();
+ return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
+ }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
index c58b5169fc..f280d27509 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
@@ -29,6 +29,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.Constant;
import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.lock.TransactionDeadLockHandler;
import org.apache.kylin.common.persistence.metadata.EpochStore;
import org.apache.kylin.common.persistence.metadata.JdbcAuditLogStore;
import org.apache.kylin.common.persistence.transaction.EventListenerRegistry;
@@ -135,6 +136,7 @@ public class AppInitializer {
EventBusFactory.getInstance().register(new StreamingJobListener(), true);
SparkJobFactoryUtils.initJobFactory();
+ TransactionDeadLockHandler.getInstance().start();
} else {
val auditLogStore = new JdbcAuditLogStore(kylinConfig);
val epochStore = EpochStore.getEpochStore(kylinConfig);
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java
index 3dece15a36..b433cf43ad 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/ModelBrokenListener.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
import org.apache.kylin.job.manager.JobManager;
import org.apache.kylin.job.model.JobParam;
import org.apache.kylin.metadata.cube.model.NDataSegment;
@@ -41,9 +43,6 @@ import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
import org.apache.kylin.metadata.sourceusage.SourceUsageManager;
import org.springframework.util.CollectionUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -94,9 +93,10 @@ public class ModelBrokenListener {
dfUpdate.setToRemoveSegs(dataflow.getSegments().toArray(new NDataSegment[0]));
}
dataflowManager.updateDataflow(dfUpdate);
- model.setHandledAfterBroken(true);
- model.setRecommendationsCount(0);
- modelManager.updateDataBrokenModelDesc(model);
+ NDataModel copy = modelManager.copyForWrite(model);
+ copy.setHandledAfterBroken(true);
+ copy.setRecommendationsCount(0);
+ modelManager.updateDataBrokenModelDesc(copy);
OptRecManagerV2 optRecManagerV2 = OptRecManagerV2.getInstance(project);
optRecManagerV2.discardAll(model.getId());
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java
index c9d1b69e9a..de0dfeecae 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/KylinUserService.java
@@ -36,6 +36,7 @@ import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -47,8 +48,6 @@ import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.util.CollectionUtils;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.metadata.user.NKylinUserManager;
import lombok.SneakyThrows;
@@ -126,8 +125,9 @@ public class KylinUserService implements UserService {
public UserDetails loadUserByUsername(String userName) throws UsernameNotFoundException {
Message msg = MsgPicker.getMsg();
ManagedUser managedUser;
+ NKylinUserManager manager = getKylinUserManager();
try {
- managedUser = getKylinUserManager().get(userName);
+ managedUser = manager.get(userName);
} catch (IllegalArgumentException e) {
log.error("exception: ", e);
throw new UsernameNotFoundException(USER_LOGIN_FAILED.getMsg());
@@ -136,7 +136,7 @@ public class KylinUserService implements UserService {
throw new UsernameNotFoundException(String.format(Locale.ROOT, msg.getUserNotFound(), userName));
}
log.trace("load user : {}", userName);
- return managedUser;
+ return manager.copy(managedUser);
}
@Override
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 4299f0d7e3..95c7578f22 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -19,13 +19,13 @@
package org.apache.kylin.rest.service;
import static org.apache.kylin.common.constant.Constants.HIDDEN_VALUE;
+import static org.apache.kylin.common.constant.Constants.KYLIN_JOB_MAX_CONCURRENT_JOBS;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_DRIVER_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_ENABLE_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_NAME_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_USER_KEY;
-import static org.apache.kylin.common.constant.Constants.KYLIN_JOB_MAX_CONCURRENT_JOBS;
import static org.apache.kylin.common.constant.NonCustomProjectLevelConfig.DATASOURCE_TYPE;
import static org.apache.kylin.common.exception.ServerErrorCode.DATABASE_NOT_EXIST;
import static org.apache.kylin.common.exception.ServerErrorCode.DUPLICATE_PROJECT_NAME;
@@ -970,8 +970,7 @@ public class ProjectService extends BasicService {
if (uppderDB.equals(projectInstance.getDefaultDatabase())) {
return;
}
- projectInstance.setDefaultDatabase(uppderDB);
- prjManager.updateProject(projectInstance);
+ prjManager.updateProject(project, copyForWrite -> copyForWrite.setDefaultDatabase(uppderDB));
} else {
throw new KylinException(DATABASE_NOT_EXIST,
String.format(Locale.ROOT, MsgPicker.getMsg().getDatabaseNotExist(), defaultDatabase));
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
index e983bcb46c..59cf7d5543 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
@@ -22,6 +22,7 @@ import static org.apache.kylin.common.exception.ServerErrorCode.CONFIG_NONEXIST_
import static org.apache.kylin.common.exception.ServerErrorCode.DIAG_FAILED;
import static org.apache.kylin.common.exception.ServerErrorCode.DIAG_UUID_NOT_EXIST;
import static org.apache.kylin.common.exception.ServerErrorCode.FILE_NOT_EXIST;
+import static org.apache.kylin.common.exception.code.ErrorCodeSystem.NOT_DEADLOCK_THREAD_IDS;
import static org.apache.kylin.tool.constant.DiagTypeEnum.FULL;
import static org.apache.kylin.tool.constant.DiagTypeEnum.JOB;
import static org.apache.kylin.tool.constant.DiagTypeEnum.QUERY;
@@ -29,10 +30,14 @@ import static org.apache.kylin.tool.constant.StageEnum.DONE;
import java.io.File;
import java.io.IOException;
+import java.lang.management.ThreadInfo;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -42,6 +47,7 @@ import java.util.stream.Collectors;
import javax.validation.constraints.NotNull;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -49,11 +55,16 @@ import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.KylinTimeoutException;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.persistence.lock.DeadLockInfo;
+import org.apache.kylin.common.persistence.lock.TransactionDeadLockHandler;
import org.apache.kylin.common.persistence.transaction.MessageSynchronization;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.guava30.shaded.common.cache.Cache;
+import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -79,9 +90,6 @@ import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
-import org.apache.kylin.guava30.shaded.common.cache.Cache;
-import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
-
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.val;
@@ -115,7 +123,8 @@ public class SystemService extends BasicService {
}
}
- private final Cache<String, DiagInfo> diagMap = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS).build();
+ private final Cache<String, DiagInfo> diagMap = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS)
+ .build();
private final Cache<String, DiagStatusResponse> exceptionMap = CacheBuilder.newBuilder()
.expireAfterAccess(1, TimeUnit.DAYS).build();
private final ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -123,7 +132,7 @@ public class SystemService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#backupRequest.getProject(), 'ADMINISTRATION')")
public void backup(BackupRequest backupRequest) throws Exception {
String project = StringUtils.isNotBlank(backupRequest.getProject()) ? backupRequest.getProject() : null;
- String path = StringUtils.isNotBlank(backupRequest.getBackupPath()) ? backupRequest.getBackupPath(): null;
+ String path = StringUtils.isNotBlank(backupRequest.getBackupPath()) ? backupRequest.getBackupPath() : null;
boolean compress = backupRequest.isCompress();
helper.backup(getConfig(), project, path, null, compress, false);
}
@@ -395,4 +404,29 @@ public class SystemService extends BasicService {
RoutineToolHelper.cleanEventLog(true, true, false);
}
}
+
+ public List<DeadLockInfo> detectDeadLock() {
+ Set<ThreadInfo> threadInfoSet = TransactionDeadLockHandler.getInstance().getThreadsToBeKill();
+ if (CollectionUtils.isEmpty(threadInfoSet)) {
+ return Collections.emptyList();
+ }
+ return threadInfoSet.stream().map(DeadLockInfo::new).collect(Collectors.toList());
+ }
+
+ public void killDeadLockThread(List<Long> ids) {
+ TransactionDeadLockHandler deadLockHandler = TransactionDeadLockHandler.getInstance();
+
+ Set<Long> threadIdSet = Sets.newHashSet(ids);
+ threadIdSet.removeAll(deadLockHandler.getThreadIdToBeKill());
+ if (CollectionUtils.isNotEmpty(threadIdSet)) {
+ throw new KylinException(NOT_DEADLOCK_THREAD_IDS, StringUtils.join(threadIdSet, ","));
+ }
+ deadLockHandler.killThreadsById(Sets.newHashSet(ids));
+ }
+
+ public void killAllDeadLockThread() {
+ TransactionDeadLockHandler deadLockHandler = TransactionDeadLockHandler.getInstance();
+ Set<Long> threadIdToBeKill = deadLockHandler.getThreadIdToBeKill();
+ deadLockHandler.killThreadsById(threadIdToBeKill);
+ }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java
index 86e5d286ba..386c8791ff 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/QueryHistoryTaskScheduler.java
@@ -137,9 +137,10 @@ public class QueryHistoryTaskScheduler {
QueryHistoryIdOffsetManager manager = QueryHistoryIdOffsetManager.getInstance(config, project);
QueryHistoryIdOffset queryHistoryIdOffset = manager.get();
if (queryHistoryIdOffset.getOffset() > maxId || queryHistoryIdOffset.getStatMetaUpdateOffset() > maxId) {
- queryHistoryIdOffset.setOffset(maxId);
- queryHistoryIdOffset.setStatMetaUpdateOffset(maxId);
- manager.save(queryHistoryIdOffset);
+ manager.updateOffset(copyForWrite -> {
+ copyForWrite.setOffset(maxId);
+ copyForWrite.setStatMetaUpdateOffset(maxId);
+ });
}
return 0;
}, project);
@@ -230,10 +231,9 @@ public class QueryHistoryTaskScheduler {
updateLastQueryTime(modelsLastQueryTime, project);
// update id offset
- QueryHistoryIdOffset queryHistoryIdOffset = QueryHistoryIdOffsetManager
- .getInstance(KylinConfig.getInstanceFromEnv(), project).get();
- queryHistoryIdOffset.setStatMetaUpdateOffset(maxId);
- QueryHistoryIdOffsetManager.getInstance(config, project).save(queryHistoryIdOffset);
+ QueryHistoryIdOffsetManager manager = QueryHistoryIdOffsetManager
+ .getInstance(config, project);
+ manager.updateOffset(copyForWrite -> copyForWrite.setStatMetaUpdateOffset(maxId));
// update snpashot hit count
incQueryHitSnapshotCount(hitSnapshotCountMap, project);
@@ -424,10 +424,9 @@ public class QueryHistoryTaskScheduler {
KylinConfig config = KylinConfig.getInstanceFromEnv();
// update id offset
- QueryHistoryIdOffset queryHistoryIdOffset = QueryHistoryIdOffsetManager
- .getInstance(KylinConfig.getInstanceFromEnv(), project).get();
- queryHistoryIdOffset.setOffset(maxId);
- QueryHistoryIdOffsetManager.getInstance(config, project).save(queryHistoryIdOffset);
+ QueryHistoryIdOffsetManager manager = QueryHistoryIdOffsetManager
+ .getInstance(config, project);
+ manager.updateOffset(copyForWrite -> copyForWrite.setOffset(maxId));
return 0;
}, project);
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java
index 6e590b6f66..044ef27ac1 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/task/RecommendationTopNUpdateScheduler.java
@@ -29,14 +29,15 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.metrics.MetricsCategory;
+import org.apache.kylin.common.metrics.MetricsGroup;
+import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.util.ExecutorServiceUtil;
import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.common.util.TimeUtil;
-import org.apache.kylin.common.metrics.MetricsCategory;
-import org.apache.kylin.common.metrics.MetricsGroup;
-import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.metadata.favorite.AsyncAccelerationTask;
+import org.apache.kylin.metadata.favorite.AsyncTaskManager;
import org.apache.kylin.metadata.favorite.FavoriteRule;
import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
@@ -146,10 +147,11 @@ public class RecommendationTopNUpdateScheduler {
public void saveTaskTime(String project) {
long currentTime = System.currentTimeMillis();
EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) getInstance(
- KylinConfig.getInstanceFromEnv(), project).get(ASYNC_ACCELERATION_TASK);
- asyncAcceleration.setLastUpdateTonNTime(currentTime);
- getInstance(KylinConfig.getInstanceFromEnv(), project).save(asyncAcceleration);
+ AsyncTaskManager manager = getInstance(KylinConfig.getInstanceFromEnv(), project);
+ AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) manager.get(ASYNC_ACCELERATION_TASK);
+ AsyncAccelerationTask copied = manager.copyForWrite(asyncAcceleration);
+ copied.setLastUpdateTonNTime(currentTime);
+ getInstance(KylinConfig.getInstanceFromEnv(), project).save(copied);
return null;
}, project);
}
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
index 9e70136557..a505364ef3 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
@@ -33,7 +33,6 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.favorite.AccelerateRuleUtil;
import org.apache.kylin.metadata.favorite.AsyncAccelerationTask;
import org.apache.kylin.metadata.favorite.AsyncTaskManager;
-import org.apache.kylin.metadata.favorite.QueryHistoryIdOffset;
import org.apache.kylin.metadata.favorite.QueryHistoryIdOffsetManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -144,10 +143,10 @@ public class QueryHistoryTaskSchedulerTest extends NLocalFileMetadataTestCase {
// before update id offset
QueryHistoryIdOffsetManager idOffsetManager = QueryHistoryIdOffsetManager.getInstance(getTestConfig(), PROJECT);
- QueryHistoryIdOffset queryHistoryIdOffset = idOffsetManager.get();
- queryHistoryIdOffset.setOffset(999L);
- queryHistoryIdOffset.setStatMetaUpdateOffset(999L);
- idOffsetManager.save(queryHistoryIdOffset);
+ idOffsetManager.updateOffset(copyForWrite -> {
+ copyForWrite.setOffset(999L);
+ copyForWrite.setStatMetaUpdateOffset(999L);
+ });
Assert.assertEquals(999L, idOffsetManager.get().getOffset());
// run update
qhAccelerateScheduler.resetOffsetId();
@@ -164,10 +163,10 @@ public class QueryHistoryTaskSchedulerTest extends NLocalFileMetadataTestCase {
Mockito.when(qhAccelerateScheduler.queryHistoryDAO.getQueryHistoryMaxId(Mockito.anyString())).thenReturn(12L);
// before update id offset
QueryHistoryIdOffsetManager idOffsetManager = QueryHistoryIdOffsetManager.getInstance(getTestConfig(), PROJECT);
- QueryHistoryIdOffset queryHistoryIdOffset = idOffsetManager.get();
- queryHistoryIdOffset.setOffset(9L);
- queryHistoryIdOffset.setStatMetaUpdateOffset(9L);
- idOffsetManager.save(queryHistoryIdOffset);
+ idOffsetManager.updateOffset(copyForWrite -> {
+ copyForWrite.setOffset(9L);
+ copyForWrite.setStatMetaUpdateOffset(9L);
+ });
Assert.assertEquals(9L, idOffsetManager.get().getOffset());
// run update
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java
similarity index 53%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java
index 1bbd33831f..b6ae0f1f0e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/util/InitResourceGroupUtilsTest.java
@@ -16,27 +16,23 @@
* limitations under the License.
*/
-package org.apache.kylin.metadata.favorite;
+package org.apache.kylin.rest.util;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.resourcegroup.ResourceGroupManager;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
-import com.fasterxml.jackson.annotation.JsonProperty;
+@MetadataInfo(onlyProps = true)
+public class InitResourceGroupUtilsTest {
+
+ @Test
+ void testInitResourceGroup() {
+ ResourceGroupManager manager = ResourceGroupManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assertions.assertFalse(manager.resourceGroupInitialized());
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ InitResourceGroupUtils.initResourceGroup();
+ Assertions.assertTrue(manager.resourceGroupInitialized());
}
}
diff --git a/src/core-common/pom.xml b/src/core-common/pom.xml
index 178418deac..a7f1426270 100644
--- a/src/core-common/pom.xml
+++ b/src/core-common/pom.xml
@@ -80,6 +80,10 @@
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-common</artifactId>
+ </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>transmittable-thread-local</artifactId>
@@ -186,6 +190,11 @@
<artifactId>jcl-over-slf4j</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f51bb8ba8a..954d13a685 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -700,6 +700,31 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.env.ip-address");
}
+ public String getTransactionLockManager() {
+ return getOptional("kylin.env.transaction-lock-manager",
+ "org.apache.kylin.common.persistence.transaction.TransactionOptimisticLockManager");
+ }
+
+ public String getLockManager() {
+ return getOptional("kylin.env.lock-manager", "org.apache.kylin.common.persistence.lock.MemoryLockManager");
+ }
+
+ public long getLockTimeoutSeconds() {
+ return Long.parseLong(getOptional("kylin.env.lock-timeout", "60"));
+ }
+
+ public long getLockCheckIntervalSeconds() {
+ return Long.parseLong(getOptional("kylin.env.dead-lock-check-interval", "3"));
+ }
+
+ public long getMaxSecondsForDeadLockRetry() {
+ return Long.parseLong(getOptional("kylin.env.max-seconds-for-dead-lock-retry", "600"));
+ }
+
+ public int getLockRemoveMaxSize() {
+ return Integer.parseInt(getOptional("kylin.env.lock-remove-max-size", "200"));
+ }
+
public String getServerAddress() {
// Caution: config 'kylin.server.address' is essential in yarn cluster mode.
// The value may be the address of loadbalancer
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java
index 12b057c677..202ea66e6c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeSystem.java
@@ -20,21 +20,27 @@ package org.apache.kylin.common.exception.code;
public enum ErrorCodeSystem implements ErrorCodeProducer {
// 400052XX password
- PASSWORD_INVALID_ENCODER("KE-040005201"), PASSWORD_INIT_ENCODER_FAILED("KE-040005202"),
+ PASSWORD_INVALID_ENCODER("KE-040005201"),
+ PASSWORD_INIT_ENCODER_FAILED("KE-040005202"),
// 400212XX epoch
EPOCH_DOES_NOT_BELONG_TO_CURRENT_NODE("KE-040021201"),
// 400232XX HA
- JOB_NODE_API_INVALID("KE-040023201"), JOB_NODE_QUERY_API_INVALID("KE-040023202"), QUERY_NODE_API_INVALID(
- "KE-040023203"),
+ JOB_NODE_API_INVALID("KE-040023201"),
+ JOB_NODE_QUERY_API_INVALID("KE-040023202"),
+ QUERY_NODE_API_INVALID("KE-040023203"),
// 400242XX maintenance mode
- MAINTENANCE_MODE_WRITE_FAILED("KE-040024201"), MAINTENANCE_MODE_ENTER_FAILED(
- "KE-040024202"), MAINTENANCE_MODE_LEAVE_FAILED("KE-040024203"),
+ MAINTENANCE_MODE_WRITE_FAILED("KE-040024201"),
+ MAINTENANCE_MODE_ENTER_FAILED("KE-040024202"),
+ MAINTENANCE_MODE_LEAVE_FAILED("KE-040024203"),
// 400252XX system config
- SYSTEM_PROFILE_ABNORMAL_DATA("KE-040026201");
+ SYSTEM_PROFILE_ABNORMAL_DATA("KE-040026201"),
+
+ // 400282XX deadlock
+ NOT_DEADLOCK_THREAD_IDS("KE-040028201");
private final ErrorCode errorCode;
private final ErrorMsg errorMsg;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java
index c2b87f1ace..26ce0935c2 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/InMemResourceStore.java
@@ -24,15 +24,14 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.FilePathUtil;
import org.apache.kylin.common.persistence.metadata.MetadataStore.MemoryMetaData;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.FilePathUtil;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.Getter;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index c720f29254..b057300025 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -41,21 +41,21 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.AuditLogStore;
import org.apache.kylin.common.persistence.metadata.EpochStore;
import org.apache.kylin.common.persistence.metadata.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.guava30.shaded.common.base.Throwables;
import org.apache.kylin.guava30.shaded.common.cache.Cache;
import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
import org.apache.kylin.guava30.shaded.common.cache.CacheLoader;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import lombok.Getter;
import lombok.Setter;
import lombok.val;
@@ -223,6 +223,11 @@ public abstract class ResourceStore implements AutoCloseable {
* Read a resource, return null in case of not found or is a folder.
*/
public final <T extends RootPersistentEntity> T getResource(String resPath, Serializer<T> serializer) {
+ return MemoryLockUtils.doWithLock(null, resPath, true, this,
+ () -> getResourceWithoutLock(resPath, serializer));
+ }
+
+ public final <T extends RootPersistentEntity> T getResourceWithoutLock(String resPath, Serializer<T> serializer) {
resPath = norm(resPath);
RawResource res = getResourceImpl(resPath);
if (res == null)
@@ -244,7 +249,7 @@ public abstract class ResourceStore implements AutoCloseable {
}
public final RawResource getResource(String resPath) {
- return getResourceImpl(norm(resPath));
+ return MemoryLockUtils.doWithLock(resPath, true, this, () -> getResourceImpl(norm(resPath)));
}
public final List<RawResource> getAllResources(String folderPath) {
@@ -484,8 +489,9 @@ public abstract class ResourceStore implements AutoCloseable {
try {
File f = Paths.get(metaDir.getAbsolutePath(), res.getResPath()).toFile();
f.getParentFile().mkdirs();
- try (FileOutputStream out = new FileOutputStream(f)) {
- IOUtils.copy(res.getByteSource().openStream(), out);
+ try (FileOutputStream out = new FileOutputStream(f);
+ InputStream in = res.getByteSource().openStream()) {
+ IOUtils.copy(in, out);
if (!f.setLastModified(res.getTimestamp())) {
logger.info("{} modified time change failed", f);
}
@@ -567,6 +573,7 @@ public abstract class ResourceStore implements AutoCloseable {
public void createMetaStoreUuidIfNotExist() {
if (!exists(METASTORE_UUID_TAG)) {
+ MemoryLockUtils.doWithLock(null, METASTORE_UUID_TAG, false, this, () -> null);
checkAndPutResource(METASTORE_UUID_TAG, new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
index d939ce6691..cde9e01b10 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/RootPersistentEntity.java
@@ -19,17 +19,19 @@
package org.apache.kylin.common.persistence;
import java.io.Serializable;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.springframework.beans.BeanUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
@@ -149,6 +151,10 @@ public abstract class RootPersistentEntity implements AclEntity, Serializable {
}
this.mvcc = mvcc;
}
+
+ public <T extends RootPersistentEntity> void copyPropertiesTo(T copy) {
+ BeanUtils.copyProperties(this, copy, "cachedAndShared");
+ }
/**
* The name as a part of the resource path used to save the entity.
@@ -192,6 +198,14 @@ public abstract class RootPersistentEntity implements AclEntity, Serializable {
return "";
}
+ public List<String> getLockPaths() {
+ return getLockPaths(getResourcePath());
+ }
+
+ public List<String> getLockPaths(String resourcePath) {
+ return Collections.singletonList(resourcePath);
+ }
+
@Override
public String toString() {
return getClass().getName() + "@" + uuid;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java
index e847e92c30..85ef794d8e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/ThreadViewResourceStore.java
@@ -25,12 +25,13 @@ import java.util.TreeSet;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.metadata.MetadataStore;
import org.apache.kylin.common.util.FilePathUtil;
-
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+
import lombok.Getter;
import lombok.val;
@@ -54,6 +55,11 @@ public class ThreadViewResourceStore extends ResourceStore {
this.overlay = new InMemResourceStore(kylinConfig);
resources = Lists.newArrayList();
}
+
+ @Override
+ public MetadataStore getMetadataStore() {
+ return underlying.getMetadataStore();
+ }
@Override
protected NavigableSet<String> listResourcesImpl(String folderPath, boolean recursive) {
@@ -106,9 +112,7 @@ public class ThreadViewResourceStore extends ResourceStore {
return r == TombRawResource.getINSTANCE() ? null //deleted
: r; // updated
}
-
return underlying.getResourceImpl(resPath);
-
}
@Override
@@ -142,6 +146,7 @@ public class ThreadViewResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) {
+ MemoryLockUtils.doWithLock(resPath, false, this, () -> null);
overlay.putTomb(resPath);
resources.add(new TombRawResource(resPath));
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java
new file mode 100644
index 0000000000..3567d92c28
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/AbstractMemoryLock.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public abstract class AbstractMemoryLock extends IntentionReadWriteLock {
+
+ private final AtomicLong tranHoldCount = new AtomicLong();
+
+ private final Long createTime;
+
+ AbstractMemoryLock(IntentionShareLock intentionShareLock) {
+ super(intentionShareLock);
+ createTime = System.currentTimeMillis();
+ }
+
+ AbstractMemoryLock(IntentionReadWriteLock intentionReadWriteLock) {
+ super(new IntentionShareLock(intentionReadWriteLock));
+ createTime = System.currentTimeMillis();
+ }
+
+ @Override
+ public Long createTime() {
+ return createTime;
+ }
+
+ public AtomicLong getTranHoldCount() {
+ return tranHoldCount;
+ }
+
+ @Override
+ public TransactionLock readLock() {
+ tranHoldCount.incrementAndGet();
+ return super.readLock();
+ }
+
+ @Override
+ public TransactionLock writeLock() {
+ tranHoldCount.incrementAndGet();
+ return super.writeLock();
+ }
+
+ public TransactionLock getReadOrWriteLock(boolean read) {
+ return read ? readLock() : writeLock();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java
index 1bbd33831f..7e3c2d9002 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockException.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+public class DeadLockException extends RuntimeException {
+ public DeadLockException(String msg) {
+ super(msg);
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java
new file mode 100644
index 0000000000..636f728368
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockHandler.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.lang.management.ThreadInfo;
+import java.util.Set;
+
+public interface DeadLockHandler {
+
+ /**
+ * Detect deadlock ThreadInfo
+ */
+ ThreadInfo[] findDeadLockThreads();
+
+ /**
+ * Get the ThreadInfo that needs to be killed
+ */
+ Set<ThreadInfo> getThreadsToBeKill();
+
+ /**
+ * Get the ThreadIds that needs to be killed
+ */
+ Set<Long> getThreadIdToBeKill();
+
+ /**
+ * Filter thread based on ThreadId
+ */
+ Set<Thread> getThreadsById(Set<Long> threadIdSet);
+
+ /**
+ * Interrupt threads
+ */
+ void killThreads(Set<Thread> threads);
+
+ /**
+ * Interrupt threads by threadIds
+ */
+ void killThreadsById(Set<Long> threadIdSet);
+
+ /**
+ * Thread name prefix for filtering
+ */
+ String getThreadNamePrefix();
+
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java
new file mode 100644
index 0000000000..b43fcc71da
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/DeadLockInfo.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.lang.management.LockInfo;
+import java.lang.management.ThreadInfo;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Data;
+
+@Data
+public class DeadLockInfo {
+
+ @JsonProperty("thread_id")
+ private long threadId;
+ @JsonProperty("thread_name")
+ private String threadName;
+ @JsonProperty("owned_locks")
+ private List<String> ownedLocks;
+ @JsonProperty("waiting_lock")
+ private String waitingLock;
+ @JsonProperty("stacktrace")
+ private String stacktrace;
+
+ @JsonIgnore
+ private static final int MAX_STACKTRACE_LENGTH = 16;
+
+ public DeadLockInfo(ThreadInfo ti) {
+ this.stacktrace = getStackTraceString(ti.getStackTrace());
+ this.threadName = ti.getThreadName();
+ this.threadId = ti.getThreadId();
+ this.waitingLock = ti.getLockInfo().toString();
+ this.ownedLocks = Arrays.stream(ti.getLockedSynchronizers()).map(LockInfo::toString)
+ .collect(Collectors.toList());
+ }
+
+ private String getStackTraceString(StackTraceElement[] st) {
+ StringBuilder sb = new StringBuilder();
+ int i = 0;
+ for (; i < st.length && i < MAX_STACKTRACE_LENGTH; i++) {
+ StackTraceElement ste = st[i];
+ sb.append("\tat ").append(ste.toString());
+ sb.append('\n');
+ }
+ if (i < st.length) {
+ sb.append("\t...");
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java
index 1bbd33831f..b063823b84 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionLock.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.concurrent.locks.Lock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public interface IntentionLock {
+ Lock shareLock();
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java
new file mode 100644
index 0000000000..ea6677cd2d
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionReadWriteLock.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import static java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.kylin.common.KylinConfig;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+
+@Slf4j
+public class IntentionReadWriteLock implements IntentionLock, ReadWriteLock, MemoryLock {
+
+ private final InnerReentrantLock lockInternal = new InnerReentrantLock();
+ private final TransactionLock readLock;
+ private final TransactionLock writeLock;
+
+ private final IntentionShareLock intentionShareLock;
+
+ public IntentionReadWriteLock(IntentionShareLock intentionShareLock) {
+ this.intentionShareLock = intentionShareLock;
+ this.readLock = new InnerReadLock(lockInternal);
+ this.writeLock = new InnerWriteLock(lockInternal);
+ }
+
+ @Override
+ public TransactionLock readLock() {
+ return readLock;
+ }
+
+ @Override
+ public TransactionLock writeLock() {
+ return writeLock;
+ }
+
+ @Override
+ public Lock shareLock() {
+ return readLock;
+ }
+
+ protected long getLockTimeoutSeconds() {
+ return KylinConfig.getInstanceFromEnv().getLockTimeoutSeconds();
+ }
+
+ private static class InnerReentrantLock extends ReentrantReadWriteLock {
+ private final Set<Long> readLockOwners = new ConcurrentSkipListSet<>();
+
+ public InnerReentrantLock() {
+ super(true);
+ }
+
+ private Set<Long> getReadLockOwners() {
+ return this.readLockOwners;
+ }
+
+ private Set<Long> getReadLockDependencyThreads(long threadId) {
+ Thread writeLockOwner = getOwner();
+ Set<Long> dependencyThreads = new HashSet<>();
+ if (writeLockOwner != null) {
+ if (writeLockOwner.getId() != threadId) {
+ dependencyThreads.add(writeLockOwner.getId());
+ }
+ } else {
+ if (!getQueuedWriterThreads().isEmpty() && !readLockOwners.contains(threadId)) {
+ dependencyThreads.addAll(readLockOwners);
+ }
+ }
+ return dependencyThreads;
+ }
+
+ private Set<Long> getWriteLockDependencyThreads(long threadId) {
+ Set<Long> dependencyThreads = new HashSet<>();
+ Thread writeLockOwner = getOwner();
+ if (writeLockOwner != null) {
+ if (writeLockOwner.getId() != threadId) {
+ dependencyThreads.add(writeLockOwner.getId());
+ }
+ } else {
+ Preconditions.checkState(!readLockOwners.contains(threadId),
+ "Must release readLock before acquire writeLock");
+ dependencyThreads.addAll(readLockOwners);
+ }
+ return dependencyThreads;
+ }
+ }
+
+ public class InnerReadLock extends ReadLock implements TransactionLock {
+
+ /**
+ * Constructor for use by subclasses
+ *
+ * @param lock the outer lock object
+ * @throws NullPointerException if the lock is null
+ */
+ protected InnerReadLock(ReentrantReadWriteLock lock) {
+ super(lock);
+ }
+
+ @Override
+ public void lock() {
+ if (!isHeldByCurrentThread() && !this.tryLock()) {
+ throw new LockTimeoutException(this.transactionUnit());
+ }
+ lockInternal.getReadLockOwners().add(Thread.currentThread().getId());
+ }
+
+ @Override
+ public boolean tryLock() {
+ try {
+ intentionShareLock.lockIntentionLock();
+ return super.tryLock(IntentionReadWriteLock.this.getLockTimeoutSeconds(), TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ intentionShareLock.unlockIntentionLock();
+ log.warn("Failed to get lock: {}. This thread is interrupted.", transactionUnit());
+ throw new LockInterruptException(e);
+ }
+ }
+
+ @Override
+ public void unlock() {
+ if (isHeldByCurrentThread()) {
+ lockInternal.getReadLockOwners().remove(Thread.currentThread().getId());
+ super.unlock();
+ intentionShareLock.unlockIntentionLock();
+ }
+ }
+
+ @Override
+ public boolean isHeldByCurrentThread() {
+ return lockInternal.getReadHoldCount() > 0;
+ }
+
+ @Override
+ public Set<Long> getLockDependencyThreads(long threadId) {
+ Set<Long> dependencyThreads = new HashSet<>();
+ dependencyThreads.addAll(intentionShareLock.getReadLockDependencyThreads(threadId));
+ dependencyThreads.addAll(lockInternal.getReadLockDependencyThreads(threadId));
+ return dependencyThreads;
+ }
+
+ @Override
+ public TransactionLock getOppositeLock() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String transactionUnit() {
+ return IntentionReadWriteLock.this.transactionUnit();
+ }
+
+ @Override
+ public Long createTime() {
+ return IntentionReadWriteLock.this.createTime();
+ }
+
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return IntentionReadWriteLock.this.moduleEnum();
+ }
+ }
+
+ public class InnerWriteLock extends ReentrantReadWriteLock.WriteLock implements TransactionLock {
+
+ /**
+ * Constructor for use by subclasses
+ *
+ * @param lock the outer lock object
+ * @throws NullPointerException if the lock is null
+ */
+ protected InnerWriteLock(ReentrantReadWriteLock lock) {
+ super(lock);
+ }
+
+ @Override
+ public void lock() {
+ if (!isHeldByCurrentThread() && !this.tryLock()) {
+ throw new LockTimeoutException(this.transactionUnit());
+ }
+ }
+
+ @Override
+ public boolean tryLock() {
+ try {
+ intentionShareLock.lockIntentionLock();
+ return super.tryLock(IntentionReadWriteLock.this.getLockTimeoutSeconds(), TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ intentionShareLock.unlockIntentionLock();
+ log.warn("Failed to get lock: {}. This thread is interrupted.", transactionUnit());
+ throw new LockInterruptException(e);
+ }
+ }
+
+ @Override
+ public void unlock() {
+ super.unlock();
+ intentionShareLock.unlockIntentionLock();
+ }
+
+ @Override
+ public boolean isHeldByCurrentThread() {
+ return lockInternal.getWriteHoldCount() > 0;
+ }
+
+ @Override
+ public Set<Long> getLockDependencyThreads(long threadId) {
+ Set<Long> dependencyThreads = new HashSet<>(intentionShareLock.getReadLockDependencyThreads(threadId));
+ dependencyThreads.addAll(lockInternal.getWriteLockDependencyThreads(threadId));
+ return dependencyThreads;
+ }
+
+ @Override
+ public TransactionLock getOppositeLock() {
+ return readLock;
+ }
+
+ @Override
+ public String transactionUnit() {
+ return IntentionReadWriteLock.this.transactionUnit();
+ }
+
+ @Override
+ public Long createTime() {
+ return IntentionReadWriteLock.this.createTime();
+ }
+
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return IntentionReadWriteLock.this.moduleEnum();
+ }
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java
new file mode 100644
index 0000000000..50633690ce
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/IntentionShareLock.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+
+public class IntentionShareLock implements IntentionLock {
+
+ private final Lock shareLock;
+
+ public IntentionShareLock() {
+ this.shareLock = null;
+ }
+
+ public IntentionShareLock(IntentionLock intentionLock) {
+ this.shareLock = intentionLock.shareLock();
+ }
+
+ @Override
+ public Lock shareLock() {
+ return shareLock;
+ }
+
+ public void lockIntentionLock() {
+ if (shareLock != null) {
+ shareLock.lock();
+ }
+ }
+
+ public void unlockIntentionLock() {
+ if (shareLock != null) {
+ shareLock.unlock();
+ }
+ }
+
+ public Set<Long> getReadLockDependencyThreads(Long threadId) {
+ if (shareLock instanceof IntentionReadWriteLock.InnerReadLock) {
+ return ((IntentionReadWriteLock.InnerReadLock) shareLock).getLockDependencyThreads(threadId);
+ }
+ return Collections.emptySet();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java
index 1bbd33831f..310ae29d7a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockInterruptException.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+public class LockInterruptException extends RuntimeException {
+ public LockInterruptException(Throwable throwable) {
+ super(throwable);
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java
similarity index 50%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java
index 86eb29c09f..180f85d9ac 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManager.java
@@ -15,30 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.List;
+import java.util.Map;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+import javax.validation.constraints.NotNull;
-import com.fasterxml.jackson.annotation.JsonProperty;
+public interface LockManager {
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ ProjectLock getProjectLock(@NotNull String projectName);
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class QueryHistoryIdOffset extends RootPersistentEntity {
- @JsonProperty("query_history_id_offset")
- private long offset;
+ ProjectLock removeProjectLock(@NotNull String projectName);
- @JsonProperty("query_history_stat_id_offset")
- private long statMetaUpdateOffset;
+ ModuleLock getModuleLock(@NotNull String projectName, @NotNull ModuleLockEnum moduleName);
- public QueryHistoryIdOffset(long offset) {
- this.offset = offset;
- }
+ /** Since metadata is in json format, when we want to lock a path, we need lock multiple resources;
+ For example, if the pathName belongs to a model, we will lock model name、cc name、cc express. **/
+ List<PathLock> getPathLock(@NotNull String pathName);
+
+ String getProjectByPath(@NotNull String pathName);
+
+ Map<String, ProjectLock> listAllProjectLock();
+
+ void checkProjectLockSize();
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java
index 1bbd33831f..e03e335ecb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockManagerInstance.java
@@ -16,27 +16,20 @@
* limitations under the License.
*/
-package org.apache.kylin.metadata.favorite;
+package org.apache.kylin.common.persistence.lock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
-import com.fasterxml.jackson.annotation.JsonProperty;
+public class LockManagerInstance {
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ public static final LockManager INSTANCE;
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
+ static {
+ INSTANCE = (LockManager) ClassUtil.newInstance(KylinConfig.getInstanceFromEnv().getLockManager());
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ private LockManagerInstance() {
}
+
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java
index 1bbd33831f..b332f2ac88 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/LockTimeoutException.java
@@ -15,28 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+public class LockTimeoutException extends RuntimeException {
+ public LockTimeoutException(String lockUnit) {
+ super(lockUnit);
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java
index 1bbd33831f..4eaf93db46 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLock.java
@@ -15,28 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+public interface MemoryLock {
+ default String transactionUnit() {
+ return UnitOfWork.GLOBAL_UNIT;
+ }
- @JsonProperty("task_type")
- private String taskType;
+ default Long createTime() {
+ return 0L;
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ default ModuleLockEnum moduleEnum() {
+ return null;
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java
new file mode 100644
index 0000000000..b8d6c66815
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockGraph.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.guava30.shaded.common.collect.ComparisonChain;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * This graph only records nodes info, the edges are calculated at realtime.
+ */
+@Slf4j
+public class MemoryLockGraph {
+
+ private final Map<Long, ThreadNode> nodeMap = Maps.newConcurrentMap();
+
+ public void setThread(long threadId, TransactionLock requiredLock) {
+ ThreadNode node = nodeMap.computeIfAbsent(threadId, ThreadNode::new);
+ node.setRequiredLock(requiredLock);
+ }
+
+ public void deleteThreadNode(long threadId) {
+ nodeMap.remove(threadId);
+ }
+
+ public void resetThread(long threadId) {
+ nodeMap.get(threadId).onLocked();
+ }
+
+ public List<Long> preCheck(long threadId, Set<Long> dependencies) {
+ if (dependencies.isEmpty()) {
+ return Collections.emptyList();
+ }
+ ThreadNode target = nodeMap.get(threadId);
+ if (target == null) {
+ return Collections.emptyList();
+ }
+ List<ThreadNode> trace = new ArrayList<>();
+ for (Long id : dependencies) {
+ ThreadNode node = nodeMap.get(id);
+ if (node == null) {
+ continue;
+ }
+ if (findCycleForTarget(target, node, trace)) {
+ return trace.stream().map(ThreadNode::getThreadId).collect(Collectors.toList());
+ }
+ }
+ return Collections.emptyList();
+ }
+
+ private boolean findCycleForTarget(ThreadNode target, ThreadNode currentNode, List<ThreadNode> trace) {
+ if (currentNode.getThreadId() == target.getThreadId()) {
+ return true;
+ }
+ if (!trace.contains(currentNode)) {
+ trace.add(currentNode);
+ for (Long threadId : currentNode.getDependencies()) {
+ ThreadNode node = nodeMap.get(threadId);
+ if (node != null && findCycleForTarget(target, node, trace)) {
+ return true;
+ }
+ }
+ trace.remove(trace.size() - 1);
+ }
+ return false;
+ }
+
+ public List<List<Long>> checkForDeadLock() {
+ List<ThreadNode> trace = new ArrayList<>();
+ List<List<Long>> cycles = new ArrayList<>();
+ for (ThreadNode startNode : nodeMap.values()) {
+ Set<Long> finishedNode = cycles.stream().flatMap(List::stream).collect(Collectors.toSet());
+ findCycles(startNode, trace, cycles, finishedNode);
+ }
+ return cycles;
+ }
+
+ private void findCycles(ThreadNode currentNode, List<ThreadNode> trace, List<List<Long>> cycles,
+ Set<Long> finishedNode) {
+ if (finishedNode.contains(currentNode.getThreadId())) {
+ return;
+ }
+ int index = trace.indexOf(currentNode);
+ if (index != -1) {
+ cycles.add(trace.subList(index, trace.size()).stream().map(ThreadNode::getThreadId)
+ .collect(Collectors.toList()));
+ } else {
+ trace.add(currentNode);
+ currentNode.getDependencies().forEach(id -> {
+ ThreadNode node = nodeMap.get(id);
+ if (node != null) {
+ findCycles(node, trace, cycles, finishedNode);
+ }
+ });
+ trace.remove(trace.size() - 1);
+ }
+ }
+
+ public Set<Long> getKeyNodes(List<List<Long>> cycles) {
+ Map<Long, Integer> cycleCntMap = Maps.newHashMap();
+ cycles.stream().flatMap(List::stream).forEach(threadId -> {
+ int cnt = cycleCntMap.getOrDefault(threadId, 0);
+ cycleCntMap.put(threadId, cnt + 1);
+ });
+ Comparator<Long> comparator = ((Comparator<Long>) (thread1, thread2) -> ComparisonChain.start()
+ .compare(cycleCntMap.get(thread1), cycleCntMap.get(thread2))
+ .compare(nodeMap.get(thread1).getCreateTime(), nodeMap.get(thread2).getCreateTime()).result())
+ .reversed();
+ Set<Long> keyNode = new HashSet<>();
+ cycles.forEach(cycle -> {
+ cycle.sort(comparator);
+ keyNode.add(cycle.get(0));
+ });
+ return keyNode;
+ }
+
+ public boolean isThreadWaitForLock(long threadId) {
+ return nodeMap.get(threadId).getRequiredLock() == null;
+ }
+
+ public int getThreadNodeCount() {
+ return nodeMap.size();
+ }
+
+ @Data
+ static class ThreadNode {
+ long threadId;
+ long createTime;
+ TransactionLock requiredLock;
+
+ private ThreadNode(long threadId) {
+ this.threadId = threadId;
+ this.createTime = System.currentTimeMillis();
+ }
+
+ private void onLocked() {
+ this.requiredLock = null;
+ }
+
+ private Set<Long> getDependencies() {
+ return requiredLock == null ? Collections.emptySet() : requiredLock.getLockDependencyThreads(threadId);
+ }
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java
new file mode 100644
index 0000000000..e6f816156a
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockManager.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.validation.constraints.NotNull;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+
+import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MemoryLockManager implements LockManager {
+ private final Map<String, ProjectLock> projectLockMap = Maps.newConcurrentMap();
+ private final Map<String, ModuleLock> moduleLockMap = Maps.newConcurrentMap();
+ private final Map<String, PathLock> pathLockMap = Maps.newConcurrentMap();
+
+ // workaround for https://bugs.openjdk.org/browse/JDK-8161372
+ private <T> T computeIfAbsent(Map<String, T> map, String key, Function<String, ? extends T> mappingFunction) {
+ T result = map.get(key);
+ if (null == result) {
+ result = map.computeIfAbsent(key, mappingFunction);
+ }
+ return result;
+ }
+
+ @Override
+ public ProjectLock getProjectLock(@NotNull String projectName) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(projectName), "projectName is blank");
+ return computeIfAbsent(projectLockMap, projectName, k -> new ProjectLock(projectName));
+ }
+
+ @Override
+ public ProjectLock removeProjectLock(@NotNull String projectName) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(projectName), "projectName is blank");
+ return projectLockMap.remove(projectName);
+ }
+
+ @Override
+ public ModuleLock getModuleLock(@NotNull String projectName, @NotNull ModuleLockEnum moduleName) {
+ Preconditions.checkNotNull(moduleName, "moduleName is null");
+ return computeIfAbsent(moduleLockMap, projectName + ":" + moduleName,
+ k -> new ModuleLock(moduleName, getProjectLock(projectName)));
+ }
+
+ @Override
+ public List<PathLock> getPathLock(@NotNull String pathName) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(pathName), "pathName is blank");
+ // parse the Module in the Path, and return the DEFAULT Module if it cannot be parsed
+ val resourcePath = ResourcePathParser.parseResourcePath(pathName);
+ ModuleLockEnum moduleLockEnum = ModuleLockEnum.getModuleEnum(resourcePath.getModule());
+
+ ModuleLock moduleLock = getModuleLock(resourcePath.getProject(), moduleLockEnum);
+ List<String> newPaths = ResourcePathParser.transformPath(pathName, moduleLockEnum, resourcePath);
+ return newPaths.stream().map(path -> computeIfAbsent(pathLockMap, path, k -> new PathLock(path, moduleLock)))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public String getProjectByPath(@NotNull String pathName) {
+ return ResourcePathParser.parseResourcePath(pathName).getProject();
+ }
+
+ @Override
+ public Map<String, ProjectLock> listAllProjectLock() {
+ return Maps.newHashMap(projectLockMap);
+ }
+
+ @Override
+ public void checkProjectLockSize() {
+ if (projectLockMap.size() > 5000L) {
+ projectLockMap.forEach((k, v) -> log.warn("lock leaks lock: {}，num: {}", k, v.getTranHoldCount()));
+ }
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java
new file mode 100644
index 0000000000..32c610427f
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/MemoryLockUtils.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import static org.apache.kylin.common.persistence.transaction.TransactionManagerInstance.INSTANCE;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.RawResource;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.persistence.ThreadViewResourceStore;
+import org.apache.kylin.common.persistence.metadata.JdbcMetadataStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MemoryLockUtils {
+
+ private MemoryLockUtils() {
+ }
+
+ private static final MemoryLockGraph MEMORY_LOCK_GRAPH = new MemoryLockGraph();
+
+ public static List<TransactionLock> getProjectLock(String project) {
+ return Collections.singletonList(INSTANCE.getProjectLock(project));
+ }
+
+ public static List<TransactionLock> getModuleLocks(String project, ModuleLockEnum module) {
+ return getModuleLocks(project, Collections.singletonList(module));
+ }
+
+ public static List<TransactionLock> getModuleLocks(String project, Collection<ModuleLockEnum> modules) {
+ return modules.stream().filter(Objects::nonNull).map(module -> INSTANCE.getModuleLock(project, module))
+ .distinct().collect(Collectors.toList());
+ }
+
+ public static List<TransactionLock> getPathLocks(String path) {
+ return getPathLocks(path, false);
+ }
+
+ public static List<TransactionLock> getPathLocks(String path, boolean readOnly) {
+ return getPathLocks(Collections.singletonList(path), readOnly);
+ }
+
+ public static List<TransactionLock> getPathLocks(Collection<String> paths) {
+ return getPathLocks(paths, false);
+ }
+
+ public static List<TransactionLock> getPathLocks(Collection<String> paths, boolean readOnly) {
+ return paths.stream().filter(StringUtils::isNotBlank).map(path -> INSTANCE.getPathLock(path, readOnly))
+ .flatMap(List::stream).distinct().collect(Collectors.toList());
+ }
+
+ public static <T extends RootPersistentEntity> T doWithLock(T entity, String resPath, boolean readonly,
+ ResourceStore store, Action<T> action) {
+ if (isNoNeedToLock(readonly, store)) {
+ return action.run();
+ }
+ if (entity == null) {
+ MemoryLockUtils.lockAndRecord(resPath, null, readonly);
+ return action.run();
+ }
+
+ List<String> lockPaths = readonly? Collections.singletonList(resPath) : entity.getLockPaths(resPath);
+ MemoryLockUtils.lockAndRecord(resPath, lockPaths, readonly);
+
+ T newEntity = action.run();
+
+ if (!readonly && newEntity != null) {
+ // update locks with new entity
+ List<String> newLockPaths = newEntity.getLockPaths(resPath);
+ List<String> delta = newLockPaths.stream().filter(path -> !lockPaths.contains(path))
+ .collect(Collectors.toList());
+ if (!delta.isEmpty()) {
+ MemoryLockUtils.lockAndRecord(resPath, delta, false);
+ }
+ }
+ return newEntity;
+ }
+
+ public static boolean isNoNeedToLock(boolean readonly, ResourceStore store) {
+ return !UnitOfWork.isAlreadyInTransaction()
+ || !readonly && !(store instanceof ThreadViewResourceStore)
+ || !(store.getMetadataStore() instanceof JdbcMetadataStore)
+ && !KylinConfig.getInstanceFromEnv().isUTEnv()
+ || UnitOfWork.get().getParams().isUseProjectLock();
+ }
+
+ public static RawResource doWithLock(String resPath, boolean readonly, ResourceStore store,
+ Action<RawResource> action) {
+ if (!isNoNeedToLock(readonly, store)) {
+ lockAndRecord(resPath, null, readonly);
+ }
+ return action.run();
+ }
+
+ public static void manuallyLockModule(String project, ModuleLockEnum module, ResourceStore store) {
+ if (isNoNeedToLock(false, store)) {
+ return;
+ }
+ List<TransactionLock> moduleLocks = getModuleLocks(project, module);
+ assert moduleLocks.size() == 1;
+ batchLock(moduleLocks.get(0).transactionUnit(), moduleLocks, false);
+ }
+
+ public static void lockAndRecord(String resPath, List<String> specifiedPath, boolean readonly){
+
+ if (resPath.equals("/")) {
+ log.warn("The resPath of Entity is wrong. Cannot lock this path.", new Throwable());
+ return;
+ }
+
+ List<String> paths = specifiedPath == null ? Collections.singletonList(resPath) : specifiedPath;
+
+ if (readonly) {
+ Set<String> holdReadLocks = UnitOfWork.get().getReadLockPath();
+ paths = paths.stream().filter(path -> !holdReadLocks.contains(path)).collect(Collectors.toList());
+ }
+
+ if (paths.isEmpty()) {
+ return;
+ }
+ List<TransactionLock> locks = getPathLocks(paths, readonly);
+
+ batchLock(resPath, locks, readonly);
+ }
+
+ private static void batchLock(String resPath, List<TransactionLock> locks, boolean readonly) {
+ for (TransactionLock lock : locks) {
+ if (lock instanceof IntentionReadWriteLock.InnerWriteLock) {
+ TransactionLock readLock = lock.getOppositeLock();
+ readLock.unlock();
+ }
+ }
+
+ locks.forEach(lock -> {
+ lockWithPreCheck(lock);
+ UnitOfWork.recordLocks(resPath, lock, readonly);
+ });
+ }
+
+ private static void lockWithPreCheck(TransactionLock lock) {
+ if (lock instanceof IntentionReadWriteLock.InnerWriteLock) {
+ TransactionLock readLock = lock.getOppositeLock();
+ readLock.unlock();
+ }
+ long threadID = Thread.currentThread().getId();
+ Set<Long> dependencyThreads = lock.getLockDependencyThreads(threadID);
+ List<Long> cycle = MEMORY_LOCK_GRAPH.preCheck(threadID, dependencyThreads);
+
+ if (cycle.isEmpty()) {
+ MEMORY_LOCK_GRAPH.setThread(threadID, lock);
+ lock.lock();
+ MEMORY_LOCK_GRAPH.resetThread(threadID);
+ } else {
+ StringBuilder builder = new StringBuilder();
+ Thread.getAllStackTraces().keySet().stream().filter(t -> cycle.contains(t.getId()))
+ .forEach(thread -> builder.append(Arrays.toString(thread.getStackTrace())));
+ throw new DeadLockException(
+ "Lock preCheck failed! To avoid deadlock, restart this transaction." + builder.toString());
+ }
+ }
+
+ public static Set<Long> findDeadLockThreadIds() {
+ List<List<Long>> cycles = MEMORY_LOCK_GRAPH.checkForDeadLock();
+ return MEMORY_LOCK_GRAPH.getKeyNodes(cycles);
+ }
+
+ public static void removeThreadFromGraph() {
+ MEMORY_LOCK_GRAPH.deleteThreadNode(Thread.currentThread().getId());
+ }
+
+ public static int getTransactionSize() {
+ return MEMORY_LOCK_GRAPH.getThreadNodeCount();
+ }
+
+ public interface Action<T> {
+ T run();
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java
similarity index 51%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java
index dc22af1698..ff7629335a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLock.java
@@ -15,33 +15,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.metadata.usergroup;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
+package org.apache.kylin.common.persistence.lock;
import lombok.Getter;
-import lombok.Setter;
-@SuppressWarnings("serial")
@Getter
-@Setter
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class UserGroup extends RootPersistentEntity {
- @JsonProperty("group_name")
- private String groupName;
+public class ModuleLock extends AbstractMemoryLock {
+
+ private final ModuleLockEnum moduleName;
+ private final ProjectLock projectLock;
+
+ public ModuleLock(ModuleLockEnum moduleName, ProjectLock projectLock) {
+ super(projectLock);
+ this.moduleName = moduleName;
+ this.projectLock = projectLock;
+ }
- public UserGroup() {
+ @Override
+ public String transactionUnit() {
+ return projectLock.transactionUnit() + ":" + moduleName;
}
- public UserGroup(String groupName) {
- this.groupName = groupName;
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return moduleName;
+ }
+
+ @Override
+ public TransactionLock readLock() {
+ throw new UnsupportedOperationException("module don's support read lock!");
}
@Override
- public String resourceName() {
- return groupName;
+ protected long getLockTimeoutSeconds() {
+ return super.getLockTimeoutSeconds() * 3;
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java
new file mode 100644
index 0000000000..9caa06ab53
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ModuleLockEnum.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_EXPRESSION;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_NAME;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.DATAFLOW;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.DATAFLOW_DETAILS;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.EXECUTE;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.INDEX_PLAN;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.MODEL_DESC;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.TABLE_EXD;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+
+public enum ModuleLockEnum {
+ JOB(EXECUTE.value()), //
+ MODEL(MODEL_DESC.value(), DATAFLOW.value(), DATAFLOW_DETAILS.value(), INDEX_PLAN.value()), //
+ TABLE(ResourceEnum.TABLE.value(), TABLE_EXD.value()), //
+ HISTORY_SOURCE_USAGE(ResourceEnum.HISTORY_SOURCE_USAGE.value()), //
+ PROJECT(ResourceEnum.PROJECT.value()), //
+ COMPUTE_COLUMN(COMPUTE_COLUMN_NAME.value(), COMPUTE_COLUMN_EXPRESSION.value()), //
+ DEFAULT();
+
+ private final List<String> moduleNames;
+
+ public List<String> getModuleNames() {
+ return moduleNames;
+ }
+
+ ModuleLockEnum(String... moduleNames) {
+ this.moduleNames = Lists.newArrayList(moduleNames);
+ }
+
+ private static final Map<String, ModuleLockEnum> MODULE_LOCK_ENUMS = Maps.newHashMap();
+
+ static {
+ Arrays.stream(values()).forEach(moduleLockEnum -> moduleLockEnum.moduleNames
+ .forEach(module -> MODULE_LOCK_ENUMS.put(module, moduleLockEnum)));
+ }
+
+ public static ModuleLockEnum getModuleEnum(String moduleName) {
+ if (StringUtils.isBlank(moduleName)) {
+ return DEFAULT;
+ }
+ return MODULE_LOCK_ENUMS.getOrDefault(moduleName, DEFAULT);
+ }
+
+ public enum ResourceEnum {
+ EXECUTE,
+ MODEL_DESC,
+ DATAFLOW,
+ DATAFLOW_DETAILS,
+ INDEX_PLAN,
+ TABLE,
+ TABLE_EXD,
+ HISTORY_SOURCE_USAGE,
+ PROJECT,
+ COMPUTE_COLUMN_NAME,
+ COMPUTE_COLUMN_EXPRESSION;
+
+ public String value() {
+ return name().toLowerCase();
+ }
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java
index 1bbd33831f..c6388cf2cb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/OptimisticLockRetryException.java
@@ -15,28 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public class OptimisticLockRetryException extends RuntimeException {
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java
index 1bbd33831f..15d7120b30 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/PathLock.java
@@ -15,28 +15,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+public class PathLock extends AbstractMemoryLock implements TempLock {
+
+ private final String pathName;
+ private final ModuleLock moduleLock;
- @JsonProperty("task_type")
- private String taskType;
+ public PathLock(String pathName, ModuleLock moduleLock) {
+ super(moduleLock);
+ this.pathName = pathName;
+ this.moduleLock = moduleLock;
+ }
+
+ @Override
+ public String transactionUnit() {
+ return moduleLock.transactionUnit() + ":" + pathName;
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ @Override
+ public ModuleLockEnum moduleEnum() {
+ return moduleLock.getModuleName();
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java
similarity index 58%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java
index 1bbd33831f..d928abbf68 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ProjectLock.java
@@ -15,28 +15,32 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import lombok.Getter;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+@Getter
+public class ProjectLock extends AbstractMemoryLock {
-import com.fasterxml.jackson.annotation.JsonProperty;
+ private final String projectName;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ public ProjectLock(String projectUnit) {
+ super(new IntentionShareLock());
+ this.projectName = projectUnit;
+ }
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+ @Override
+ public String transactionUnit() {
+ return projectName;
+ }
- @JsonProperty("task_type")
- private String taskType;
+ @Override
+ public TransactionLock readLock() {
+ throw new UnsupportedOperationException("project don's support read lock!");
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ @Override
+ protected long getLockTimeoutSeconds() {
+ return super.getLockTimeoutSeconds() * 10;
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java
new file mode 100644
index 0000000000..14ecfa021b
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/ResourcePathParser.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.DEFAULT;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.guava30.shaded.common.base.Joiner;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.cache.Cache;
+import org.apache.kylin.guava30.shaded.common.cache.CacheBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.glassfish.jersey.uri.UriTemplate;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+@UtilityClass
+@Slf4j
+public class ResourcePathParser {
+
+ private static final int CACHE_SIZE = 10000;
+ private static final int CACHE_EXPIRE_MINUTES = 30;
+ private static final String RESOURCE_PATH_SUFFIX = ".json";
+ private static final Cache<String, ResourcePath> RES_CACHE = CacheBuilder.newBuilder().maximumSize(CACHE_SIZE)
+ .expireAfterAccess(CACHE_EXPIRE_MINUTES, TimeUnit.MINUTES).build();
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final List<String> TEMPLATE_URIS;
+ private static final List<UriTemplate> TEMPLATES;
+ public static final Joiner JOINER = Joiner.on('/').skipNulls();
+
+ static {
+ TEMPLATE_URIS = Lists.newArrayList(//
+ "/{project}/{module}/{resource1}", // normal
+ "/{project}/{module}/{resource1}/{resource2}", // dataflow_detail
+ "/{project}/{module}", // _health
+ "/{project}" // UUID
+ );
+ TEMPLATES = TEMPLATE_URIS.stream().map(UriTemplate::new).collect(Collectors.toList());
+ }
+
+ public static ResourcePath parseResourcePath(String originPath) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(originPath), "path is blank");
+ if (originPath.endsWith(RESOURCE_PATH_SUFFIX)) {
+ originPath = originPath.substring(0, originPath.length() - RESOURCE_PATH_SUFFIX.length());
+ }
+
+ ResourcePath resourcePath = RES_CACHE.getIfPresent(originPath);
+ if (Objects.nonNull(resourcePath)) {
+ return resourcePath;
+ }
+
+ for (UriTemplate template : TEMPLATES) {
+ Map<String, String> map = Maps.newHashMap();
+ if (template.match(originPath, map)) {
+ resourcePath = MAPPER.convertValue(map, ResourcePath.class);
+ RES_CACHE.put(originPath, resourcePath);
+ return resourcePath;
+ }
+ }
+ throw new IllegalArgumentException("The path does not match: " + originPath);
+ }
+
+ public static List<String> transformPath(String originPath, ModuleLockEnum moduleLockEnum,
+ ResourcePathParser.ResourcePath resourcePath) {
+ if (moduleLockEnum.equals(DEFAULT) || Objects.isNull(resourcePath)) {
+ // can not parse module
+ return Collections.singletonList(originPath);
+ }
+
+ // transform path name
+ return resourcePath.transformPath();
+ }
+
+ @AllArgsConstructor
+ @NoArgsConstructor
+ @Data
+ public static class ResourcePath {
+ private String project;
+ private String module;
+ private String resource1;
+ private String resource2;
+
+ public List<String> transformPath() {
+ return getDefaultLockPath();
+ }
+
+ private List<String> getDefaultLockPath() {
+ return Collections.singletonList(JOINER.join(project, module, resource1, resource2));
+ }
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java
index 1bbd33831f..8803befff8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TempLock.java
@@ -15,28 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public interface TempLock {
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java
new file mode 100644
index 0000000000..275c337fc2
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionDeadLockHandler.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.NamedThreadFactory;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.guava30.shaded.common.primitives.Longs;
+
+@Slf4j
+public class TransactionDeadLockHandler implements DeadLockHandler, Runnable {
+
+ public static final String THREAD_NAME_PREFIX = "Transaction-Thread";
+ private final ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+ private final AtomicBoolean scheduleStarted = new AtomicBoolean(false);
+ private final long interval = KylinConfig.getInstanceFromEnv().getLockCheckIntervalSeconds();
+ private final ScheduledExecutorService checkerPool = Executors.newScheduledThreadPool(1,
+ new NamedThreadFactory("DeadLockChecker"));
+
+ private int checkCnt = 0;
+ private double checkAvgCost = 0;
+ private long checkMaxCost = 0;
+
+ private TransactionDeadLockHandler() {
+ }
+
+ private static final class InstanceHolder {
+ static final TransactionDeadLockHandler INSTANCE = new TransactionDeadLockHandler();
+ }
+
+ public static TransactionDeadLockHandler getInstance() {
+ return InstanceHolder.INSTANCE;
+ }
+
+ public void start() {
+ if (scheduleStarted.compareAndSet(false, true)) {
+ log.info("Start scheduling deadlock detection every {} seconds", interval);
+ checkerPool.scheduleWithFixedDelay(this, 1, interval, TimeUnit.SECONDS);
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ Set<ThreadInfo> threadInfoSet = getThreadsToBeKill();
+ if (CollectionUtils.isEmpty(threadInfoSet)) {
+ return;
+ }
+
+ for (ThreadInfo threadInfo : threadInfoSet) {
+ log.warn("Deadlock thread detail:\n{}", threadInfo);
+ }
+ Set<Long> threadIdSet = threadInfoSet.stream().map(ThreadInfo::getThreadId).collect(Collectors.toSet());
+ killThreadsById(threadIdSet);
+ } catch (Throwable throwable) {
+ log.info("Deadlock detection failed. Wait for next execution ...", throwable);
+ }
+ }
+
+ @Override
+ public Set<ThreadInfo> getThreadsToBeKill() {
+ ThreadInfo[] threadInfos = findDeadLockThreads();
+ if (ArrayUtils.isEmpty(threadInfos)) {
+ return Collections.emptySet();
+ }
+ Set<ThreadInfo> threadInfoSet = Arrays.stream(threadInfos)//
+ .filter(Objects::nonNull)//
+ .filter(t -> StringUtils.startsWith(t.getThreadName(), getThreadNamePrefix()))
+ .collect(Collectors.toSet());
+ if (CollectionUtils.isEmpty(threadInfoSet)) {
+ return Collections.emptySet();
+ }
+ log.info("Found transaction deadlock thread, size {}, total transaction size: {}", threadInfoSet.size(),
+ MemoryLockUtils.getTransactionSize());
+ return threadInfoSet;
+ }
+
+ @Override
+ public Set<Long> getThreadIdToBeKill() {
+ return getThreadsToBeKill().stream().filter(Objects::nonNull).map(ThreadInfo::getThreadId)
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public ThreadInfo[] findDeadLockThreads() {
+ long start = System.currentTimeMillis();
+ long[] threadIds = Longs.toArray(MemoryLockUtils.findDeadLockThreadIds());
+ updateTimeStatistics(System.currentTimeMillis() - start);
+ if (ArrayUtils.isEmpty(threadIds)) {
+ return new ThreadInfo[0];
+ }
+ return mxBean.getThreadInfo(threadIds, true, true);
+ }
+
+ private void updateTimeStatistics(long timeCost) {
+ checkAvgCost = (checkAvgCost * checkCnt + timeCost) / (checkCnt + 1);
+ checkCnt = checkCnt + 1;
+ checkMaxCost = Math.max(checkMaxCost, timeCost);
+ if (checkCnt % 200 == 0) {
+ log.debug(
+ "DeadLock checker has run {} times, the average cost time is {} ms, and the max cost time is {} ms.",
+ checkCnt, checkAvgCost, checkMaxCost);
+ }
+ }
+
+ @Override
+ public Set<Thread> getThreadsById(Set<Long> threadIdSet) {
+ if (CollectionUtils.isEmpty(threadIdSet)) {
+ return Collections.emptySet();
+ }
+ return Thread.getAllStackTraces().keySet().stream().filter(t -> threadIdSet.contains(t.getId()))
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public void killThreads(Set<Thread> threads) {
+ for (Thread thread : threads) {
+ log.info("Interrupt thread: {}", thread.getName());
+ thread.interrupt();
+ }
+ }
+
+ @Override
+ public void killThreadsById(Set<Long> threadIdSet) {
+ if (CollectionUtils.isEmpty(threadIdSet)) {
+ return;
+ }
+ killThreads(getThreadsById(threadIdSet));
+ }
+
+ @Override
+ public String getThreadNamePrefix() {
+ return THREAD_NAME_PREFIX;
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java
similarity index 59%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java
index 1bbd33831f..115228c940 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLock.java
@@ -15,28 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
- }
+public interface TransactionLock extends Lock, MemoryLock {
+ boolean isHeldByCurrentThread();
+ Set<Long> getLockDependencyThreads(long threadId);
+ TransactionLock getOppositeLock();
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java
similarity index 51%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java
index 86eb29c09f..fdade6e10b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionLockManager.java
@@ -15,30 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import java.util.List;
+import java.util.Map;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public interface TransactionLockManager {
-import com.fasterxml.jackson.annotation.JsonProperty;
+ TransactionLock getLock(String project, boolean readonly);
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+ TransactionLock getTempLock(String project, boolean readonly);
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class QueryHistoryIdOffset extends RootPersistentEntity {
- @JsonProperty("query_history_id_offset")
- private long offset;
+ void removeLock(String project);
- @JsonProperty("query_history_stat_id_offset")
- private long statMetaUpdateOffset;
+ Map<String, ProjectLock> getProjectLocksForRead();
- public QueryHistoryIdOffset(long offset) {
- this.offset = offset;
- }
+ TransactionLock getProjectLock(String projectName);
+
+ TransactionLock getModuleLock(String projectName, ModuleLockEnum moduleName);
+
+ /** Since metadata is in json format, when we want to lock a path, we need lock multiple resources;
+ For example, if the pathName belongs to a model, we will lock model name、cc name、cc express. **/
+ List<TransactionLock> getPathLock(String pathName, boolean readOnly);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java
index 1bbd33831f..6bc98e9d20 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionProjectLock.java
@@ -15,28 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import lombok.experimental.Delegate;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionProjectLock implements TransactionLock {
-import com.fasterxml.jackson.annotation.JsonProperty;
+ @Delegate
+ private TransactionLock lock;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public TransactionProjectLock(TransactionLock lock) {
+ this.lock = lock;
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java
index 1bbd33831f..ca9a14a117 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/TransactionTempLock.java
@@ -15,28 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock;
-package org.apache.kylin.metadata.favorite;
+import lombok.experimental.Delegate;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionTempLock implements TransactionLock, TempLock {
+ @Delegate
+ private TransactionLock lock;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public TransactionTempLock(TransactionLock lock) {
+ this.lock = lock;
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java
index 1bbd33831f..d41ccabf63 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRule.java
@@ -15,28 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock.rule;
-package org.apache.kylin.metadata.favorite;
+public abstract class LockEliminationRule {
+ private LockEliminationRule next;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
+ public void setNext(LockEliminationRule next) {
+ this.next = next;
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public void handle(LockInfo lockInfo) {
+ this.doHandle(lockInfo);
+ if (next != null) {
+ next.handle(lockInfo);
+ }
}
+
+ protected abstract void doHandle(LockInfo lockInfo);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java
new file mode 100644
index 0000000000..d2bc22bf99
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExec.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+
+public class LockEliminationRuleExec {
+ private final LockEliminationRule ruleChain;
+
+ private LockEliminationRuleExec() {
+ // projectRule -> moduleRule -> pathRule
+ ProjectEliminationRule projectRule = new ProjectEliminationRule();
+ ModuleEliminationRule moduleRule = new ModuleEliminationRule();
+ PathEliminationRule pathRule = new PathEliminationRule();
+ moduleRule.setNext(pathRule);
+ projectRule.setNext(moduleRule);
+ ruleChain = projectRule;
+ }
+
+ private static final class InstanceHolder {
+ static final LockEliminationRuleExec INSTANCE = new LockEliminationRuleExec();
+ }
+
+ public static LockEliminationRuleExec getInstance() {
+ return InstanceHolder.INSTANCE;
+ }
+
+ /**
+ * Execute the lock elimination rule chain.
+ */
+ public LockInfo execute(String project, List<TransactionLock> projectLocks, List<TransactionLock> moduleLocks,
+ List<TransactionLock> pathLocks) {
+ LockInfo lockInfo = new LockInfo(project, projectLocks, moduleLocks, pathLocks);
+ if (projectLocks.isEmpty() && moduleLocks.isEmpty() && pathLocks.isEmpty()) {
+ return lockInfo;
+ }
+ ruleChain.handle(lockInfo);
+ return lockInfo;
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java
new file mode 100644
index 0000000000..426f6b2b04
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/LockInfo.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+
+import lombok.Data;
+
+@Data
+public class LockInfo {
+ private String project;
+ private List<TransactionLock> projectLocks;
+ private List<TransactionLock> moduleLocks;
+ private List<TransactionLock> pathLocks;
+
+ public LockInfo(String project, List<TransactionLock> projectLocks, List<TransactionLock> moduleLocks,
+ List<TransactionLock> pathLocks) {
+ this.project = project;
+ this.projectLocks = projectLocks;
+ this.moduleLocks = moduleLocks;
+ this.pathLocks = pathLocks;
+ }
+
+ public List<TransactionLock> getAllLocks() {
+ List<TransactionLock> result = Lists.newArrayList();
+ result.addAll(projectLocks);
+ result.addAll(moduleLocks);
+ result.addAll(pathLocks);
+ return result;
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java
new file mode 100644
index 0000000000..34df00cc27
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ModuleEliminationRule.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * If the path locks are included by module locks, the path locks will be cleared.
+ */
+@Slf4j
+public class ModuleEliminationRule extends LockEliminationRule {
+ @Override
+ protected void doHandle(LockInfo lockInfo) {
+ if (lockInfo.getPathLocks().isEmpty() || lockInfo.getModuleLocks().isEmpty()) {
+ return;
+ }
+
+ List<ModuleLockEnum> modules = lockInfo.getModuleLocks().stream().map(TransactionLock::moduleEnum)
+ .collect(Collectors.toList());
+ Map<ModuleLockEnum, List<TransactionLock>> modulePathMap = lockInfo.getPathLocks().stream()
+ .collect(Collectors.groupingBy(TransactionLock::moduleEnum));
+ modulePathMap.forEach((module, pathLocks) -> {
+ if (modules.contains(module)) {
+ pathLocks.clear();
+ log.info("path lock has been included by module lock [{}]", module);
+ }
+ });
+ List<TransactionLock> newPathLocks = modulePathMap.values().stream().filter(CollectionUtils::isNotEmpty)
+ .flatMap(List::stream).collect(Collectors.toList());
+ lockInfo.setPathLocks(newPathLocks);
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java
new file mode 100644
index 0000000000..4f878839cb
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/PathEliminationRule.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * If the number of path locks exceeds the maximum size
+ * the path locks will be cleared and the corresponding module locks will be added.
+ */
+@Slf4j
+public class PathEliminationRule extends LockEliminationRule {
+
+ @Override
+ protected void doHandle(LockInfo lockInfo) {
+ if (lockInfo.getPathLocks().isEmpty()) {
+ return;
+ }
+
+ int maxSize = KylinConfig.getInstanceFromEnv().getLockRemoveMaxSize();
+ Map<ModuleLockEnum, List<TransactionLock>> modulePathMap = lockInfo.getPathLocks().stream()
+ .collect(Collectors.groupingBy(TransactionLock::moduleEnum));
+
+ List<ModuleLockEnum> moduleNames = Lists.newArrayList();
+ modulePathMap.forEach((module, pathLocks) -> {
+ if (pathLocks.size() > maxSize) {
+ pathLocks.clear();
+ moduleNames.add(module);
+ }
+ });
+
+ if (moduleNames.isEmpty()) {
+ return;
+ }
+ log.info("add new module locks: [{}]", StringUtils.join(moduleNames, ","));
+ List<TransactionLock> newPathLocks = modulePathMap.values().stream().filter(CollectionUtils::isNotEmpty)
+ .flatMap(List::stream).collect(Collectors.toList());
+ List<TransactionLock> newModuleLocks = MemoryLockUtils.getModuleLocks(lockInfo.getProject(), moduleNames);
+
+ lockInfo.setPathLocks(newPathLocks);
+ lockInfo.getModuleLocks().addAll(newModuleLocks);
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java
similarity index 58%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java
index 1bbd33831f..2e0ca80c39 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/lock/rule/ProjectEliminationRule.java
@@ -15,28 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.lock.rule;
-package org.apache.kylin.metadata.favorite;
+import lombok.extern.slf4j.Slf4j;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+/**
+ * If the project locks is empty, then the module locks and path locks will be cleared.
+ */
+@Slf4j
+public class ProjectEliminationRule extends LockEliminationRule {
+ @Override
+ protected void doHandle(LockInfo lockInfo) {
+ if (lockInfo.getProjectLocks().isEmpty()) {
+ return;
+ }
+ lockInfo.getModuleLocks().clear();
+ lockInfo.getPathLocks().clear();
+ log.info("the project locks is not empty, clean up the module locks and path locks");
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
index 4a119bf3e5..d8c5c2e33d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcMetadataStore.java
@@ -99,6 +99,7 @@ public class JdbcMetadataStore extends MetadataStore {
@Getter
private final JdbcTemplate jdbcTemplate;
private final String table;
+ private final boolean isUT;
public JdbcMetadataStore(KylinConfig config) throws Exception {
super(config);
@@ -108,6 +109,7 @@ public class JdbcMetadataStore extends MetadataStore {
transactionManager = new DataSourceTransactionManager(dataSource);
jdbcTemplate = new JdbcTemplate(dataSource);
table = url.getIdentifier();
+ isUT = config.isUTEnv();
if (config.isMetadataAuditLogEnabled()) {
auditLogStore = new JdbcAuditLogStore(config, jdbcTemplate, transactionManager,
table + JdbcAuditLogStore.AUDIT_LOG_SUFFIX);
@@ -129,6 +131,7 @@ public class JdbcMetadataStore extends MetadataStore {
String.format(Locale.ROOT, SELECT_BY_KEY_MVCC_SQL, table, path, mvcc - 1),
RAW_RESOURCE_ROW_MAPPER);
if (CollectionUtils.isEmpty(result)) {
+ assert isUT || mvcc == 0;
affectedRow = insert(String.format(Locale.ROOT, INSERT_SQL, table), path, bs, ts, mvcc);
} else {
affectedRow = update(String.format(Locale.ROOT, UPDATE_SQL, table), bs, mvcc, ts, path,
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java
index 1fc9c6a7a6..f802d32e95 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/MetadataStore.java
@@ -89,6 +89,7 @@ public abstract class MetadataStore {
public void batchUpdate(UnitMessages unitMessages, boolean skipAuditLog, String unitPath, long epochId)
throws Exception {
+ UnitOfWork.get().onStartUnitUpdate();
for (Event event : unitMessages.getMessages()) {
if (event instanceof ResourceCreateOrUpdateEvent) {
val rawResource = ((ResourceCreateOrUpdateEvent) event).getCreatedOrUpdated();
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java
index 263e020cff..3877a191da 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/MessageSynchronization.java
@@ -27,7 +27,6 @@ import org.apache.kylin.common.persistence.UnitMessages;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
import org.apache.kylin.common.scheduler.EventBusFactory;
-
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Setter;
@@ -115,7 +114,7 @@ public class MessageSynchronization {
}
public void replayAllMetadata(boolean needCloseReplay) throws IOException {
- val lockKeys = Lists.newArrayList(TransactionLock.getProjectLocksForRead().keySet());
+ val lockKeys = Lists.newArrayList(TransactionManagerInstance.INSTANCE.getProjectLocksForRead().keySet());
lockKeys.sort(Comparator.naturalOrder());
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
try {
@@ -124,7 +123,7 @@ public class MessageSynchronization {
resourceStore.getAuditLogStore().pause();
}
for (String lockKey : lockKeys) {
- TransactionLock.getLock(lockKey, false).lock();
+ TransactionManagerInstance.INSTANCE.getProjectLock(lockKey).lock();
}
log.info("Acquired all locks, start to reload");
@@ -133,7 +132,7 @@ public class MessageSynchronization {
} finally {
Collections.reverse(lockKeys);
for (String lockKey : lockKeys) {
- TransactionLock.getLock(lockKey, false).unlock();
+ TransactionManagerInstance.INSTANCE.getProjectLock(lockKey).unlock();
}
if (needCloseReplay) {
// if not stop, reinit return directly
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java
deleted file mode 100644
index 6807331cf2..0000000000
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionLock.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.persistence.transaction;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import lombok.experimental.Delegate;
-
-public class TransactionLock {
-
- static Map<String, Pair<ReentrantReadWriteLock, AtomicLong>> projectLocks = Maps.newConcurrentMap();
- private static final Logger logger = LoggerFactory.getLogger(TransactionLock.class);
-
- public static TransactionLock getLock(String project, boolean readonly) {
- Pair<ReentrantReadWriteLock, AtomicLong> lockPair = projectLocks.get(project);
- if (lockPair == null) {
- synchronized (UnitOfWork.class) {
- lockPair = projectLocks.computeIfAbsent(project,
- k -> Pair.newPair(new ReentrantReadWriteLock(true), new AtomicLong()));
- }
- }
- lockPair.getSecond().incrementAndGet();
- ReentrantReadWriteLock lock = lockPair.getFirst();
- return new TransactionLock(lock, readonly ? lock.readLock() : lock.writeLock());
- }
-
- public static void removeLock(String project) {
- Pair<ReentrantReadWriteLock, AtomicLong> lockPair = projectLocks.get(project);
- if (lockPair != null) {
- synchronized (UnitOfWork.class) {
- AtomicLong atomicLong = lockPair.getSecond();
- if (atomicLong.decrementAndGet() == 0) {
- projectLocks.remove(project);
- }
-
- }
- }
- if (projectLocks.size() > 5000L) {
- projectLocks.forEach((k, v) -> logger.warn("lock leaks lock: {}，num: {}", k, v.getSecond()));
- }
- }
-
- public static boolean isWriteLocked(String project) {
- ReentrantReadWriteLock lock = projectLocks.get(project).getFirst();
- return lock != null && lock.isWriteLocked();
- }
-
- private ReentrantReadWriteLock rootLock;
-
- @Delegate
- private Lock lock;
-
- public TransactionLock(ReentrantReadWriteLock rootLock, Lock lock) {
- this.rootLock = rootLock;
- this.lock = lock;
- }
-
- boolean isHeldByCurrentThread() {
- if (lock instanceof ReentrantReadWriteLock.ReadLock) {
- return rootLock.getReadHoldCount() > 0;
- } else {
- return rootLock.getWriteHoldCount() > 0;
- }
- }
-
- public static Map<String, ReentrantReadWriteLock> getProjectLocksForRead() {
- Map<String, ReentrantReadWriteLock> map = Maps.newConcurrentMap();
- projectLocks.forEach((k, v) -> map.put(k, v.getFirst()));
- return Collections.unmodifiableMap(map);
- }
-}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionManagerInstance.java
similarity index 55%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionManagerInstance.java
index 1bbd33831f..267815c501 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionManagerInstance.java
@@ -15,28 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.transaction;
-package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.TransactionLockManager;
+import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionManagerInstance {
+ public static final TransactionLockManager INSTANCE;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
+ static {
+ INSTANCE = (TransactionLockManager) ClassUtil
+ .newInstance(KylinConfig.getInstanceFromEnv().getTransactionLockManager());
+ }
- @JsonProperty("task_type")
- private String taskType;
+ private TransactionManagerInstance() {
+ }
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ public static boolean optimisticEnabled() {
+ return INSTANCE instanceof TransactionOptimisticLockManager;
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java
similarity index 60%
copy from src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
copy to src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java
index 1bbd33831f..9128075b6c 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionOptimisticLockManager.java
@@ -15,28 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.persistence.transaction;
-package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
+public class TransactionOptimisticLockManager extends TransactionPessimisticLockManager {
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@Getter
-@Setter
-@NoArgsConstructor
-@EqualsAndHashCode
-public class AbstractAsyncTask extends RootPersistentEntity {
-
- @JsonProperty("task_type")
- private String taskType;
-
- public AbstractAsyncTask(String taskType) {
- this.taskType = taskType;
+ @Override
+ public TransactionLock getLock(String project, boolean readonly) {
+ return super.getTempLock(String.valueOf(Thread.currentThread().getId()), readonly);
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java
new file mode 100644
index 0000000000..33ff6fbf3f
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/TransactionPessimisticLockManager.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.transaction;
+
+import static org.apache.kylin.common.persistence.lock.LockManagerInstance.INSTANCE;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.persistence.lock.AbstractMemoryLock;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
+import org.apache.kylin.common.persistence.lock.ProjectLock;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.common.persistence.lock.TransactionLockManager;
+import org.apache.kylin.common.persistence.lock.TransactionTempLock;
+
+public class TransactionPessimisticLockManager implements TransactionLockManager {
+
+ @Override
+ public TransactionLock getLock(String project, boolean readonly) {
+ return INSTANCE.getProjectLock(project).getReadOrWriteLock(false);
+ }
+
+ @Override
+ public TransactionLock getTempLock(String project, boolean readonly) {
+ AbstractMemoryLock lock = INSTANCE.getProjectLock(project);
+ return new TransactionTempLock(lock.writeLock());
+ }
+
+ @Override
+ public void removeLock(String project) {
+ AbstractMemoryLock lock = INSTANCE.getProjectLock(project);
+ if (lock != null) {
+ synchronized (UnitOfWork.class) {
+ AtomicLong atomicLong = lock.getTranHoldCount();
+ if (atomicLong.decrementAndGet() == 0) {
+ INSTANCE.removeProjectLock(project);
+ }
+ }
+ }
+ INSTANCE.checkProjectLockSize();
+ }
+
+ public Map<String, ProjectLock> getProjectLocksForRead() {
+ return Collections.unmodifiableMap(INSTANCE.listAllProjectLock());
+ }
+
+ @Override
+ public TransactionLock getProjectLock(String projectName) {
+ return INSTANCE.getProjectLock(projectName).getReadOrWriteLock(false);
+ }
+
+ @Override
+ public TransactionLock getModuleLock(String projectName, ModuleLockEnum moduleName) {
+ return INSTANCE.getModuleLock(projectName, moduleName).getReadOrWriteLock(false);
+ }
+
+ @Override
+ public List<TransactionLock> getPathLock(String pathName, boolean readOnly) {
+ return INSTANCE.getPathLock(pathName).stream().map(pathLock -> pathLock.getReadOrWriteLock(readOnly))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
index f344deb055..f1acfcd5a8 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
@@ -21,6 +21,14 @@ import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
+import static org.apache.kylin.common.persistence.lock.TransactionDeadLockHandler.THREAD_NAME_PREFIX;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.LogConstant;
@@ -39,16 +47,19 @@ import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
import org.apache.kylin.common.persistence.event.ResourceRelatedEvent;
import org.apache.kylin.common.persistence.event.StartUnit;
+import org.apache.kylin.common.persistence.lock.DeadLockException;
+import org.apache.kylin.common.persistence.lock.LockInterruptException;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.TempLock;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.common.persistence.metadata.JdbcMetadataStore;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
@Slf4j
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
@@ -77,33 +88,38 @@ public class UnitOfWork {
}
public static <T> T doInTransactionWithRetry(UnitOfWorkParams<T> params) {
- val maxRetry = params.getMaxRetry();
- val f = params.getProcessor();
- // reused transaction, won't retry
- if (isAlreadyInTransaction()) {
- val unitOfWork = UnitOfWork.get();
- unitOfWork.checkReentrant(params);
- try {
- checkEpoch(params);
- f.preProcess();
- return f.process();
- } catch (Throwable throwable) {
- f.onProcessError(throwable);
- throw new TransactionException("transaction failed due to inconsistent state", throwable);
+ try (SetThreadName ignore = new SetThreadName(THREAD_NAME_PREFIX)) {
+ val f = params.getProcessor();
+ // reused transaction, won't retry
+ if (isAlreadyInTransaction()) {
+ val unitOfWork = UnitOfWork.get();
+ unitOfWork.checkReentrant(params);
+ try {
+ checkEpoch(params);
+ f.preProcess();
+ return f.process();
+ } catch (Throwable throwable) {
+ f.onProcessError(throwable);
+ throw new TransactionException("transaction failed due to inconsistent state", throwable);
+ }
}
- }
- // new independent transaction with retry
- int retry = 0;
- val traceId = RandomUtil.randomUUIDStr();
- while (retry++ < maxRetry) {
+ // new independent transaction with retry
+ int retry = 0;
+ val traceId = RandomUtil.randomUUIDStr();
+ if (params.isRetryMoreTimeForDeadLockException()) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ params.setRetryUntil(System.currentTimeMillis() + config.getMaxSecondsForDeadLockRetry() * 1000);
+ }
+ while (retry++ < params.getMaxRetry()) {
- val ret = doTransaction(params, retry, traceId);
- if (ret.getSecond()) {
- return ret.getFirst();
+ val ret = doTransaction(params, retry, traceId);
+ if (ret.getSecond()) {
+ return ret.getFirst();
+ }
}
+ throw new IllegalStateException("Unexpected doInTransactionWithRetry end");
}
- throw new IllegalStateException("Unexpected doInTransactionWithRetry end");
}
private static <T> Pair<T, Boolean> doTransaction(UnitOfWorkParams<T> params, int retry, String traceId) {
@@ -144,11 +160,15 @@ public class UnitOfWork {
if (isAlreadyInTransaction()) {
try (SetLogCategory ignored = new SetLogCategory(LogConstant.METADATA_CATEGORY)) {
val unitOfWork = UnitOfWork.get();
- unitOfWork.getCurrentLock().unlock();
+ MemoryLockUtils.removeThreadFromGraph();
+ List<TransactionLock> lockInOrder = Arrays
+ .asList(unitOfWork.getCurrentLock().toArray(new TransactionLock[0]));
+ Lists.reverse(lockInOrder).stream().filter(TransactionLock::isHeldByCurrentThread)
+ .forEach(TransactionLock::unlock);
unitOfWork.cleanResource();
- if (params.getTempLockName() != null && !params.getTempLockName().equals(params.getUnitName())) {
- TransactionLock.removeLock(params.getTempLockName());
- }
+ val curLock = UnitOfWork.threadLocals.get().getCurrentLock();
+ curLock.stream().filter(TempLock.class::isInstance)
+ .forEach(l -> TransactionManagerInstance.INSTANCE.removeLock(l.transactionUnit()));
} catch (IllegalStateException e) {
//has not hold the lock yet, it's ok
log.warn(e.getMessage());
@@ -179,23 +199,10 @@ public class UnitOfWork {
}
static <T> UnitOfWorkContext startTransaction(UnitOfWorkParams<T> params) throws Exception {
- val project = params.getUnitName();
+ val unitName = params.getUnitName();
val readonly = params.isReadonly();
checkEpoch(params);
- val lock = params.getTempLockName() == null ? TransactionLock.getLock(project, readonly)
- : TransactionLock.getLock(params.getTempLockName(), readonly);
-
- try (SetLogCategory ignored = new SetLogCategory(LogConstant.METADATA_CATEGORY)) {
- log.trace("get lock for project {}, lock is held by current thread: {}", project, lock.isHeldByCurrentThread());
- }
- //re-entry is not encouraged (because it indicates complex handling logic, bad smell), let's abandon it first
- Preconditions.checkState(!lock.isHeldByCurrentThread());
- lock.lock();
-
- val unitOfWork = new UnitOfWorkContext(project);
- unitOfWork.setCurrentLock(lock);
- unitOfWork.setParams(params);
- threadLocals.set(unitOfWork);
+ final UnitOfWorkContext unitOfWork = initUnitOfContext(params, unitName, readonly);
if (readonly || !params.isUseSandbox()) {
unitOfWork.setLocalConfig(null);
@@ -218,6 +225,34 @@ public class UnitOfWork {
return unitOfWork;
}
+ private static <T> UnitOfWorkContext initUnitOfContext(UnitOfWorkParams<T> params, String unitName,
+ boolean readonly) {
+ val unitOfWork = new UnitOfWorkContext(unitName);
+ TransactionLock lock;
+ if (params.isUseProjectLock()) {
+ lock = TransactionManagerInstance.INSTANCE.getProjectLock(unitName);
+ } else if (params.getTempLockName() != null) {
+ lock = TransactionManagerInstance.INSTANCE.getTempLock(params.getTempLockName(), readonly);
+ } else {
+ lock = TransactionManagerInstance.INSTANCE.getLock(unitName, readonly);
+ if (lock.transactionUnit().equals(unitName)) {
+ params.setUseProjectLock(true);
+ }
+ }
+ try (SetLogCategory ignored = new SetLogCategory(LogConstant.METADATA_CATEGORY)) {
+ log.info("get lock for project {}, lock is held by current thread: {}", unitName,
+ lock.isHeldByCurrentThread());
+ }
+ // re-entry is not encouraged (because it indicates complex handling logic, bad smell), let's abandon it first
+ Preconditions.checkState(!lock.isHeldByCurrentThread());
+ lock.lock();
+ unitOfWork.getCurrentLock().add(lock);
+
+ unitOfWork.setParams(params);
+ threadLocals.set(unitOfWork);
+ return unitOfWork;
+ }
+
private static <T> void checkEpoch(UnitOfWorkParams<T> params) throws Exception {
val checker = params.getEpochChecker();
if (checker != null && !params.isReadonly()) {
@@ -241,6 +276,16 @@ public class UnitOfWork {
}
val threadViewRS = (ThreadViewResourceStore) ResourceStore.getKylinMetaStore(config);
List<RawResource> data = threadViewRS.getResources();
+ if (!params.isUseProjectLock()
+ && (threadViewRS.getMetadataStore() instanceof JdbcMetadataStore || config.isUTEnv())) {
+ data.forEach(rawResource -> {
+ if (!UnitOfWork.checkWriteLock(rawResource.getResPath())) {
+ throw new IllegalStateException(
+ "Transaction does not hold a write lock for resPath: " + rawResource.getResPath());
+ }
+ });
+ }
+
val eventList = data.stream().map(x -> {
if (x instanceof TombRawResource) {
return new ResourceDeleteEvent(x.getResPath());
@@ -280,12 +325,37 @@ public class UnitOfWork {
}
}
+ public static void recordLocks(String resPath, TransactionLock lock, boolean readOnly) {
+ Preconditions.checkState(lock.isHeldByCurrentThread());
+ UnitOfWork.get().getCurrentLock().add(lock);
+ if (readOnly) {
+ UnitOfWork.get().getReadLockPath().add(resPath);
+ } else {
+ UnitOfWork.get().getWriteLockPath().add(resPath);
+ }
+ }
+
+ public static boolean checkWriteLock(String resPath) {
+ return UnitOfWork.get().getWriteLockPath().contains(resPath);
+ }
+
private static void handleError(Throwable throwable, UnitOfWorkParams<?> params, int retry, String traceId) {
if (throwable instanceof KylinException && Objects.nonNull(((KylinException) throwable).getErrorCodeProducer())
&& ((KylinException) throwable).getErrorCodeProducer().getErrorCode()
.equals(ErrorCodeSystem.MAINTENANCE_MODE_WRITE_FAILED.getErrorCode())) {
retry = params.getMaxRetry();
}
+ if (throwable instanceof DeadLockException) {
+ log.debug("DeadLock found in this transaction, will retry");
+ if (params.isRetryMoreTimeForDeadLockException() && System.currentTimeMillis() < params.getRetryUntil()) {
+ params.setMaxRetry(params.getMaxRetry() + 1);
+ }
+ }
+ if (throwable instanceof LockInterruptException) {
+ // Just remove the interrupted flag.
+ Thread.interrupted();
+ log.debug("DeadLock is found by TransactionDeadLockHandler, will retry");
+ }
if (throwable instanceof QuitTxnRightNow) {
retry = params.getMaxRetry();
}
@@ -333,7 +403,6 @@ public class UnitOfWork {
public static boolean isReplaying() {
return Objects.equals(true, replaying.get());
-
}
public static boolean isReadonly() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java
index e546c62127..08083664d6 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkContext.java
@@ -17,13 +17,17 @@
*/
package org.apache.kylin.common.persistence.transaction;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.CommonErrorCode;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
-
+import org.apache.kylin.common.persistence.lock.TransactionLock;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
@@ -42,7 +46,9 @@ public class UnitOfWorkContext {
private final String project;
private KylinConfig.SetAndUnsetThreadLocalConfig localConfig;
- private TransactionLock currentLock = null;
+ private Set<TransactionLock> currentLock = new LinkedHashSet<>();
+ private Set<String> writeLockPath = new HashSet<>();
+ private Set<String> readLockPath = new HashSet<>();
@Delegate
private UnitOfWorkParams params;
@@ -51,6 +57,8 @@ public class UnitOfWorkContext {
List<UnitTask> onUpdatedTasks = Lists.newArrayList();
+ List<UnitTask> onStartUpdateTasks = Lists.newArrayList();
+
public void doAfterUnit(UnitTask task) {
onFinishedTasks.add(task);
}
@@ -59,6 +67,10 @@ public class UnitOfWorkContext {
onUpdatedTasks.add(task);
}
+ public void doBeforeUpdate(UnitTask task) {
+ onStartUpdateTasks.add(task);
+ }
+
void cleanResource() {
if (localConfig == null) {
return;
@@ -71,9 +83,9 @@ public class UnitOfWorkContext {
}
void checkLockStatus() {
- Preconditions.checkNotNull(currentLock);
- Preconditions.checkState(currentLock.isHeldByCurrentThread());
-
+ Preconditions.checkState(CollectionUtils.isNotEmpty(currentLock));
+ // Some readLocks have been released when upgrade to writeLock.
+ Preconditions.checkState(currentLock.stream().anyMatch(TransactionLock::isHeldByCurrentThread));
}
void checkReentrant(UnitOfWorkParams params) {
@@ -109,6 +121,12 @@ public class UnitOfWorkContext {
});
}
+ public void onStartUnitUpdate() throws Exception{
+ for (UnitTask task : onStartUpdateTasks) {
+ task.run();
+ }
+ }
+
public interface UnitTask {
void run() throws Exception;
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
index 57e339fe37..bb7602847d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWorkParams.java
@@ -17,12 +17,17 @@
*/
package org.apache.kylin.common.persistence.transaction;
+import java.util.List;
import java.util.function.Consumer;
import org.apache.kylin.common.persistence.event.ResourceRelatedEvent;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
@Data
@Builder
@@ -34,6 +39,10 @@ public class UnitOfWorkParams<T> {
private Consumer<ResourceRelatedEvent> writeInterceptor;
+ private UnitRetryContext retryContext;
+
+ private long retryUntil;
+
@Builder.Default
private boolean all = false;
@@ -58,6 +67,12 @@ public class UnitOfWorkParams<T> {
private boolean skipAuditLog = false;
private String tempLockName;
+
+ @Builder.Default
+ private boolean useProjectLock = false;
+
+ @Builder.Default
+ private boolean retryMoreTimeForDeadLockException = false;
/**
* only for debug or test
@@ -65,4 +80,13 @@ public class UnitOfWorkParams<T> {
@Builder.Default
private long sleepMills = -1;
+ @Getter
+ @Setter
+ @AllArgsConstructor
+ public static class UnitRetryContext {
+ private List<TransactionLock> retryLock;
+ private boolean allowRetryNext;
+ private boolean optimisticLockEnabled;
+ }
+
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index 78513f1d74..7211d629cb 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -38,6 +38,10 @@ import java.util.function.BiConsumer;
import javax.annotation.Nullable;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.type.TypeFactory;
+import com.fasterxml.jackson.databind.util.LRUMap;
+import com.fasterxml.jackson.databind.util.LookupCache;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
@@ -59,13 +63,17 @@ public class JsonUtil {
.setFailOnUnknownId(false);
static {
+ LookupCache<Object, JavaType> cache = new LRUMap<>(16, 2000);
+ TypeFactory customTypeFactory = TypeFactory.defaultInstance().withCache(cache);
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.setConfig(mapper.getSerializationConfig().withView(PersistenceView.class));
mapper.setFilterProvider(simpleFilterProvider);
+ mapper.setTypeFactory(customTypeFactory);
indentMapper.configure(SerializationFeature.INDENT_OUTPUT, true)
.setConfig(indentMapper.getSerializationConfig().withView(PersistenceView.class));
indentMapper.setFilterProvider(simpleFilterProvider);
+ indentMapper.setTypeFactory(customTypeFactory);
}
public static <T> T readValue(File src, Class<T> valueType) throws IOException {
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 2e19879659..408746be99 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -216,6 +216,9 @@ KE-040027208=参数 “%s” 不可为空, 在 “%s” 中。请检查后重试
KE-040027209=无法绑定项目 “%s” 的资源请求。请确保一个项目最多绑定两个资源组，且构建和查询请求各绑定一个资源组。
KE-040027210=在 “mapping_info” 中参数 “resource_group_id” 的值 “%s”，与 “resource_groups” 中的值不匹配。
+## 400282XX deadlock
+KE-040028201=“%s” 不是死锁线程 ID。
+
# Tool
## 500402XX
KE-050040201=参数 “%s” 为空。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index f9d66504ca..4bdfe174ba 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -216,6 +216,9 @@ KE-040027208=The parameter "%s" cannot be empty in "%s". Please check and try ag
KE-040027209=Can't allocate resource group for project "%s". Please ensure that the project is allocated to two resource groups at most. Meanwhile, each request (query or build) is allocated to one resource group.
KE-040027210=The value "%s" of "resource_group_id" in "mapping_info" does not match the value of "resource_groups".
+## 400282XX deadlock
+KE-040028201="%s" are not deadlock thread IDs.
+
# Tool
## 500402XX
KE-050040201="%s" is empty.
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index f8dd010445..f0162f4e95 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -222,6 +222,9 @@ KE-040027208
KE-040027209
KE-040027210
+## 400282XX deadlock
+KE-040028201
+
# Segment
## 100222XX Segment
KE-010022218
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java
index b667b2643b..33749e8212 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcAuditLogRecoveryTest.java
@@ -27,11 +27,16 @@ import java.util.stream.IntStream;
import org.apache.commons.dbcp2.BasicDataSourceFactory;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.JdbcAuditLogStore;
import org.apache.kylin.common.persistence.transaction.AuditLogReplayWorker;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.scheduler.EventBusFactory;
+import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.base.Joiner;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.awaitility.Awaitility;
@@ -42,11 +47,6 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.DefaultTransactionDefinition;
-import org.apache.kylin.guava30.shaded.common.base.Joiner;
-
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.eventbus.Subscribe;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -97,12 +97,14 @@ public class JdbcAuditLogRecoveryTest {
val txManager = auditLogStore.getTransactionManager();
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/_global/project/p1.json", null, false);
store.checkAndPutResource("/_global/project/p1.json", new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
return null;
}, "p1");
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/_global/project/p2.json", null, false);
store.checkAndPutResource("/_global/project/p2.json", new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
return null;
@@ -119,6 +121,7 @@ public class JdbcAuditLogRecoveryTest {
Thread.sleep(500);
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
val path = "/p1/abc-" + System.currentTimeMillis();
+ MemoryLockUtils.lockAndRecord(path, null, false);
val originAbc = store.getResource(path);
store.checkAndPutResource(path, ByteSource.wrap("abc".getBytes(charset)),
System.currentTimeMillis(), originAbc == null ? -1 : originAbc.getMvcc());
@@ -127,7 +130,7 @@ public class JdbcAuditLogRecoveryTest {
try {
auditLogStore.catchupWithTimeout();
} catch (Exception e) {
- log.debug("catchup 1st phase failed", e);
+ log.error("catchup 1st phase failed", e);
}
});
t1.start();
@@ -178,13 +181,14 @@ public class JdbcAuditLogRecoveryTest {
try {
auditLogStore.catchupWithTimeout();
} catch (Exception e) {
- log.debug("catchup 2nd phase failed", e);
+ log.error("catchup 2nd phase failed", e);
}
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
IntStream.range(1000, 1000 + size).forEach(id -> {
String path = "/p2/abc" + id;
+ MemoryLockUtils.lockAndRecord(path, null, false);
val originAbc = store.getResource(path);
store.checkAndPutResource(path, ByteSource.wrap((path + "-version2").getBytes(charset)),
System.currentTimeMillis(), originAbc == null ? -1 : originAbc.getMvcc());
@@ -194,10 +198,10 @@ public class JdbcAuditLogRecoveryTest {
try {
auditLogStore.catchupWithTimeout();
} catch (Exception e) {
- log.debug("catchup 3rd phase failed", e);
+ log.error("catchup 3rd phase failed", e);
}
}).start();
- Awaitility.await().atMost(20, TimeUnit.SECONDS).until(() -> listener.status == -1);
+ Awaitility.await().atMost(200, TimeUnit.SECONDS).until(() -> listener.status == -1);
Assert.assertEquals(203, systemStore.listResourcesRecursively("/").size());
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java
index 696791f601..ee48dc8b4e 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/JdbcMetadataStoreTest.java
@@ -31,28 +31,32 @@ import java.util.stream.IntStream;
import org.apache.commons.dbcp2.BasicDataSourceFactory;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.persistence.event.Event;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.MetadataStore;
import org.apache.kylin.common.persistence.metadata.jdbc.RawResourceRowMapper;
+import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.junit.Assert;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
+import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
@MetadataInfo(onlyProps = true)
@OverwriteProp(key = "kylin.metadata.url", value = "test@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1,username=sa,password=")
-public class JdbcMetadataStoreTest {
+class JdbcMetadataStoreTest {
public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
@@ -63,9 +67,13 @@ public class JdbcMetadataStoreTest {
}
@Test
- public void testBasic() throws IOException {
+ void testBasic() throws IOException {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc2", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc3", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc4", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
store.checkAndPutResource("/p1/abc2", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
store.checkAndPutResource("/p1/abc3", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
@@ -97,9 +105,10 @@ public class JdbcMetadataStoreTest {
}
@Test
- public void testReload() throws Exception {
+ void testReload() throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/_global/abc", null, false);
store.checkAndPutResource("/_global/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
return 0;
}, "_global");
@@ -119,7 +128,7 @@ public class JdbcMetadataStoreTest {
}
@Test
- public void testPage() throws Exception {
+ void testPage() throws Exception {
val dataSource = new DriverManagerDataSource();
val url = getTestConfig().getMetadataUrl();
dataSource.setUrl(url.getParameter("url"));
@@ -156,9 +165,10 @@ public class JdbcMetadataStoreTest {
@Test
@Disabled("for develop")
- public void testDuplicate() {
+ void testDuplicate() {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), -1);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), 0);
return 0;
@@ -175,13 +185,14 @@ public class JdbcMetadataStoreTest {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(DEFAULT_CHARSET)), 1);
return 0;
}, "p1");
}
@Test
- public void testBatchUpdate() throws Exception {
+ void testBatchUpdate() throws Exception {
val metadataStore = MetadataStore.createMetadataStore(getTestConfig());
List<Event> events = Collections.singletonList(new ResourceCreateOrUpdateEvent(new RawResource("/p1/test",
ByteSource.wrap("test content".getBytes(StandardCharsets.UTF_8)), System.currentTimeMillis(), 0)));
@@ -223,12 +234,26 @@ public class JdbcMetadataStoreTest {
CompressionUtils.decompress(auditLogContents));
}
+ @Test
+ void testUpdateTheDeletedResource() {
+ val metadataStore = MetadataStore.createMetadataStore(getTestConfig());
+ ReflectionTestUtils.setField(metadataStore, "isUT", false);
+ List<Event> events = Collections.singletonList(new ResourceCreateOrUpdateEvent(new RawResource("/p1/test",
+ ByteSource.wrap("test content".getBytes(StandardCharsets.UTF_8)), System.currentTimeMillis(), 4)));
+ val unitMessages = new UnitMessages(events);
+
+ Assertions.assertThrows(TransactionException.class, () -> UnitOfWork.doInTransactionWithRetry(() -> {
+ metadataStore.batchUpdate(unitMessages, false, "/p1/test", -1);
+ return null;
+ }, "p1", 1));
+ }
+
@OverwriteProp.OverwriteProps({ //
@OverwriteProp(key = "kylin.metadata.compress.enabled", value = "false"), //
@OverwriteProp(key = "kylin.server.port", value = "8081")//
})
@Test
- public void testBatchUpdateWithMetadataCompressDisable() throws Exception {
+ void testBatchUpdateWithMetadataCompressDisable() throws Exception {
val metadataStore = MetadataStore.createMetadataStore(getTestConfig());
List<Event> events = Collections.singletonList(new ResourceCreateOrUpdateEvent(new RawResource("/p1/test",
ByteSource.wrap("test content".getBytes(StandardCharsets.UTF_8)), System.currentTimeMillis(), 0)));
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java
index b984d67d38..44303f4781 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/UnitOfWorkTest.java
@@ -22,17 +22,19 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.DeadLockException;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
+import org.apache.kylin.guava30.shaded.common.base.Throwables;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.base.Throwables;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
@MetadataInfo(onlyProps = true)
@@ -42,19 +44,22 @@ public class UnitOfWorkTest {
public void testTransaction() {
val ret = UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res",
+ MemoryLockUtils.lockAndRecord("/_global/path/res", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res2", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res3", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res2",
+ resourceStore.checkAndPutResource("/_global/path/res2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res3",
+ resourceStore.checkAndPutResource("/_global/path/res3",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}, UnitOfWork.GLOBAL_UNIT);
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res2").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res3").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res2").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res3").getMvcc());
}
@Test
@@ -62,9 +67,11 @@ public class UnitOfWorkTest {
try {
val ret = UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res",
+ MemoryLockUtils.lockAndRecord("/_global/path/res", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res2", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res2",
+ resourceStore.checkAndPutResource("/_global/path/res2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
throw new IllegalArgumentException("surprise");
}, UnitOfWork.GLOBAL_UNIT);
@@ -72,8 +79,8 @@ public class UnitOfWorkTest {
}
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertNull(resourceStore.getResource("/_global/path/to/res"));
- Assert.assertNull(resourceStore.getResource("/_global/path/to/res2"));
+ Assert.assertNull(resourceStore.getResource("/_global/path/res"));
+ Assert.assertNull(resourceStore.getResource("/_global/path/res2"));
// test can be used again after exception
testTransaction();
@@ -132,45 +139,53 @@ public class UnitOfWorkTest {
public void testReentrant() {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res",
+ MemoryLockUtils.lockAndRecord("/_global/path/res", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path/res2", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore.checkAndPutResource("/_global/path/to/res2",
+ resourceStore.checkAndPutResource("/_global/path/res2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore2 = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore2.checkAndPutResource("/_global/path2/to/res2/1",
+ MemoryLockUtils.lockAndRecord("/_global/path2/res2/1", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path2/res2/2", null, false);
+ MemoryLockUtils.lockAndRecord("/_global/path2/res2/3", null, false);
+ resourceStore2.checkAndPutResource("/_global/path2/res2/1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore2.checkAndPutResource("/_global/path2/to/res2/2",
+ resourceStore2.checkAndPutResource("/_global/path2/res2/2",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
- resourceStore2.checkAndPutResource("/_global/path2/to/res2/3",
+ resourceStore2.checkAndPutResource("/_global/path2/res2/3",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
Assert.assertEquals(resourceStore, resourceStore2);
return 0;
}, UnitOfWork.GLOBAL_UNIT);
- resourceStore.checkAndPutResource("/_global/path/to/res3",
+ MemoryLockUtils.lockAndRecord("/_global/path/res3", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res3",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}, UnitOfWork.GLOBAL_UNIT);
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res2").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path2/to/res2/1").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path2/to/res2/2").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path2/to/res2/3").getMvcc());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res3").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res2").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path2/res2/1").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path2/res2/2").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path2/res2/3").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res3").getMvcc());
}
@Test
public void testReadLockExclusive() {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res1",
+ resourceStore.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
Object condition = new Object();
AtomicBoolean stop = new AtomicBoolean();
Thread readLockHelder = new Thread(() -> {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(true).maxRetry(1).processor(() -> {
+ ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
+ .getResource("/_global/path/res1");
synchronized (condition) {
condition.notify();
}
@@ -203,7 +218,7 @@ public class UnitOfWorkTest {
long cost = System.currentTimeMillis() - readStart;
Assert.assertTrue(cost < 500);
Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ .getResource("/_global/path/res1").getMvcc());
return 0;
}).build());
} catch (Exception e) {
@@ -221,10 +236,11 @@ public class UnitOfWorkTest {
try {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(false).maxRetry(1).processor(() -> {
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
long cost = System.currentTimeMillis() - writeStart;
Assert.assertTrue(cost > 1500);
Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ .getResource("/_global/path/res1").getMvcc());
return 0;
}).build());
} catch (Exception e) {
@@ -242,7 +258,9 @@ public class UnitOfWorkTest {
.readonly(false).maxRetry(1).processor(() -> {
val resourceStoreInTransaction = ResourceStore
.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStoreInTransaction.checkAndPutResource("/_global/path/to/res1",
+ System.out.println("Write thread start to lock");
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
+ resourceStoreInTransaction.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
synchronized (condition) {
condition.notify();
@@ -261,6 +279,7 @@ public class UnitOfWorkTest {
synchronized (condition) {
condition.notify();
}
+ System.out.println("Write thread finished.");
return 0;
}).build());
});
@@ -284,14 +303,18 @@ public class UnitOfWorkTest {
try {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(true).maxRetry(1).processor(() -> {
+ System.out.println("Read thread start to lock.");
+ RawResource raw = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
+ .getResource("/_global/path/res1");
+ System.out.println("Read thread lock succeed.");
long cost = System.currentTimeMillis() - start;
+ System.out.println(cost + " " + (raw == null ? -2 : raw.getMvcc()));
Assert.assertTrue(cost > 1500);
- Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ Assert.assertEquals(0, raw.getMvcc());
return 0;
}).build());
} catch (Exception e) {
- Assert.fail();
+ throw e;
}
stop.set(true);
}
@@ -303,7 +326,8 @@ public class UnitOfWorkTest {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT)
.readonly(true).maxRetry(1).processor(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res1",
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}).build());
@@ -315,12 +339,12 @@ public class UnitOfWorkTest {
@Test
public void testReadTransaction() {
- ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv()).checkAndPutResource("/_global/path/to/res1",
+ ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv()).checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
UnitOfWork.doInTransactionWithRetry(
UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT).readonly(true).maxRetry(1).processor(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- Assert.assertEquals(0, resourceStore.getResource("/_global/path/to/res1").getMvcc());
+ Assert.assertEquals(0, resourceStore.getResource("/_global/path/res1").getMvcc());
return 0;
}).build());
}
@@ -331,12 +355,27 @@ public class UnitOfWorkTest {
UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT).readonly(false)
.maxRetry(1).processor(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/_global/path/to/res1",
+ MemoryLockUtils.lockAndRecord("/_global/path/res1", null, false);
+ resourceStore.checkAndPutResource("/_global/path/res1",
ByteSource.wrap("{}".getBytes(Charset.defaultCharset())), -1L);
return 0;
}).build());
Assert.assertEquals(0, ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv())
- .getResource("/_global/path/to/res1").getMvcc());
+ .getResource("/_global/path/res1").getMvcc());
+
+ }
+ @Test
+ void testRetryMoreTimeForDeadLockException() {
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.env.max-seconds-for-dead-lock-retry", "2");
+ long startTime = System.currentTimeMillis();
+ try {
+ UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.builder().retryMoreTimeForDeadLockException(true).processor(() -> {
+ throw new DeadLockException("test");
+ }).build());
+ } catch (Exception e) {
+ Assert.assertEquals(DeadLockException.class, e.getCause().getClass());
+ Assert.assertTrue(System.currentTimeMillis() - startTime > 2 * 1000);
+ }
}
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java
new file mode 100644
index 0000000000..39484eba23
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/CloseWaitTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence.lock;
+
+import static org.awaitility.Awaitility.await;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.InMemResourceStore;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ThreadViewResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@MetadataInfo(onlyProps = true)
+@Disabled("Only run this manually")
+class CloseWaitTest {
+
+ private Server server = null;
+ private static ResourceStore store = null;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.env.lock-timeout", "30");
+ config.setProperty("kylin.env.dead-lock-check-interval", "10");
+
+ if (server == null) {
+ server = new Server(7077);
+ server.setHandler(new TransactionHandler());
+ server.start();
+ InMemResourceStore inMemResourceStore = (InMemResourceStore) ResourceStore.getKylinMetaStore(config);
+ store = new ThreadViewResourceStore(inMemResourceStore, config);
+ TransactionDeadLockHandler.getInstance().start();
+ }
+ }
+
+ @AfterEach
+ public void cleanup() throws Exception {
+ server.stop();
+ }
+
+ private static class TransactionHandler extends AbstractHandler {
+
+ @Override
+ public void handle(String s, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
+ throws IOException, ServletException {
+ response.setStatus(HttpServletResponse.SC_OK);
+ response.getWriter().println("Mocked response");
+ baseRequest.setHandled(true);
+ if (s.equals("/api/suggest")) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.manuallyLockModule("default", ModuleLockEnum.MODEL, store);
+ System.out.println("Suggest transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Suggest transaction failed");
+ }
+ } else if (s.equals("/api/drop")) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", false, store, () -> null);
+ System.out.println("Drop transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Drop transaction failed");
+ }
+ }
+ }
+ }
+
+ @Disabled("Only run this manually")
+ @Test
+ void closeWaitTest() {
+ AtomicBoolean shouldContinue = new AtomicBoolean(true);
+ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(5* 1000).setConnectionRequestTimeout(5 *1000)
+ .setSocketTimeout(5*1000).build();
+ Runnable suggest = () -> {
+ while (shouldContinue.get()) {
+ long start = System.currentTimeMillis();
+ try (CloseableHttpClient client = HttpClientBuilder.create()
+ .setDefaultRequestConfig(requestConfig).build()) {
+ HttpPost suggestRequest = new HttpPost("http://127.0.0.1:7077/api/suggest");
+
+ client.execute(suggestRequest);
+ } catch (IOException e) {
+ System.out.println("suggestRequest failed.");
+ } finally {
+ System.out.println("suggestRequest cost: " + (System.currentTimeMillis() - start) + "ms");
+ }
+ }
+ };
+ Runnable drop = () -> {
+ while (shouldContinue.get()) {
+ long start = System.currentTimeMillis();
+ try (CloseableHttpClient client = HttpClientBuilder.create()
+ .setDefaultRequestConfig(requestConfig).build()) {
+ HttpPost suggestRequest = new HttpPost("http://127.0.0.1:7077/api/drop");
+
+ client.execute(suggestRequest);
+ } catch (IOException e) {
+ System.out.println("dropRequest failed.");
+ } finally {
+ System.out.println("suggestRequest cost: " + (System.currentTimeMillis() - start) + "ms");
+ }
+ }
+ };
+ List<Thread> threads = new ArrayList<>();
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(drop));
+
+ threads.forEach(Thread::start);
+ await().atMost(10000, TimeUnit.SECONDS).until(() -> false);
+ }
+
+}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java
new file mode 100644
index 0000000000..c25ab05236
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/MemoryLockGraphTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock;
+
+import static org.awaitility.Awaitility.await;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.InMemResourceStore;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ThreadViewResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@MetadataInfo(onlyProps = true)
+class MemoryLockGraphTest {
+
+ @Test
+ @Disabled("A simulation of long running. When the number of CPU cores is small, it mat fail.")
+ void deadLockWithParallelThreadTest() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ InMemResourceStore inMemResourceStore = (InMemResourceStore) ResourceStore.getKylinMetaStore(config);
+ ThreadViewResourceStore store = new ThreadViewResourceStore(inMemResourceStore, config);
+ AtomicBoolean shouldContinue = new AtomicBoolean(true);
+ Runnable suggest = () -> {
+ while(shouldContinue.get()) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.manuallyLockModule("default", ModuleLockEnum.MODEL, store);
+ System.out.println("Suggest transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Suggest transaction failed");
+ }
+ }
+ };
+ Runnable drop = () -> {
+ while(shouldContinue.get()) {
+ try {
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", true, store, () -> null);
+ MemoryLockUtils.doWithLock("/default/model_desc/mDrop", false, store, () -> null);
+ System.out.println("Drop transaction succeed");
+ return null;
+ }, "default");
+ } catch (Exception e) {
+ if (Thread.currentThread().isInterrupted()) {
+ // do nothing.
+ }
+ System.out.println("Drop transaction failed");
+ }
+ }
+ };
+
+ List<Thread> threads = new ArrayList<>();
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(suggest));
+ threads.add(new Thread(drop));
+
+
+ config.setProperty("kylin.env.dead-lock-check-interval", "3");
+ TransactionDeadLockHandler.getInstance().start();
+
+ threads.forEach(Thread::start);
+
+ await().atMost(60, TimeUnit.SECONDS).until(() -> threads.stream().allMatch(t -> t.getState() == Thread.State.TIMED_WAITING));
+ await().atMost(4, TimeUnit.SECONDS).until(() -> threads.stream().anyMatch(t -> t.getState() != Thread.State.TIMED_WAITING));
+
+ shouldContinue.set(false);
+ }
+
+ @Test
+ void deadLockCheckTest() throws InterruptedException {
+ MemoryLockGraph graph = new MemoryLockGraph();
+ MockedLockManager lockManager = new MockedLockManager(5, graph);
+ lockManager.addLock(graph, "R_r5", "R_r1", "R_r2", "R_r3", "R_r3,R_r4");
+ lockManager.addLock(graph, "W_r1", "W_r2", "W_r3", "W_r4", "W_r5");
+ List<List<Long>> cycles = graph.checkForDeadLock();
+ cycles.sort((l1, l2) -> l2.size() - l1.size());
+ Assertions.assertEquals(2, cycles.size());
+ List<Long> ids = lockManager.getThreadIds();
+ Assertions.assertEquals(ids.size(), cycles.get(0).size());
+ Assertions.assertTrue(cycles.get(0).containsAll(ids));
+ Assertions.assertEquals(4, cycles.get(1).size());
+ Assertions.assertTrue(cycles.get(1).containsAll(Arrays.asList(ids.get(0), ids.get(1), ids.get(2), ids.get(4))));
+ lockManager.interruptAllThread();
+ }
+
+ @Test
+ void getKeyNodeForDeadLockTest() throws InterruptedException {
+ MemoryLockGraph graph = new MemoryLockGraph();
+ MockedLockManager lockManager = new MockedLockManager(5, graph);
+ lockManager.addLock(graph, "", "R_r4,R_r1", "R_r4,R_r2", "", "");
+ lockManager.addLock(graph, "", "", "", "R_r3,R_r5", "R_r3");
+ lockManager.addLock(graph, "W_r1", "W_r2", "W_r3", "W_r4", "W_r5");
+ List<List<Long>> cycles = graph.checkForDeadLock();
+ cycles.sort((l1, l2) -> l2.size() - l1.size());
+ Assertions.assertEquals(4, cycles.size());
+ Set<Long> nodesInCycle = graph.getKeyNodes(cycles);
+ List<Long> ids = lockManager.getThreadIds();
+ Assertions.assertArrayEquals(new Long[] { ids.get(3) }, nodesInCycle.toArray());
+ lockManager.interruptAllThread();
+ }
+
+ @Test
+ void preCheckTest() throws InterruptedException {
+ MemoryLockGraph graph = new MemoryLockGraph();
+ MockedLockManager lockManager = new MockedLockManager(5, graph);
+ lockManager.addLock(graph, "", "R_r1", "R_r2", "R_r3", "R_r3");
+ lockManager.addLock(graph, "W_r1", "W_r2", "W_r3", "", "");
+ List<Long> allThreads = lockManager.getThreadIds();
+
+ List<Long> cycle;
+ cycle = graph.preCheck(allThreads.get(4), new HashSet<>(Collections.singletonList(allThreads.get(1))));
+ Assertions.assertFalse(cycle.isEmpty());
+
+ cycle = graph.preCheck(allThreads.get(0), new HashSet<>(Collections.singletonList(allThreads.get(3))));
+ Assertions.assertTrue(cycle.isEmpty());
+
+ lockManager.interruptAllThread();
+ }
+
+ static class MockedLockManager {
+ List<Thread> threads = new ArrayList<>();
+ List<ArrayBlockingQueue<String>> lockQueues = new ArrayList<>();
+
+ public MockedLockManager(int cnt, MemoryLockGraph graph) {
+ for (int i = 0; i < cnt; i++) {
+ lockQueues.add(new ArrayBlockingQueue<>(10));
+ int finalI = i;
+ Thread t = new Thread(() -> {
+ long threadId = Thread.currentThread().getId();
+ String res;
+ List<TransactionLock> holdLocks = new ArrayList<>();
+ while (true) {
+ try {
+ res = lockQueues.get(finalI).take();
+ if (res.isEmpty()) {
+ continue;
+ }
+ String path = "/" + res.substring(2);
+ List<TransactionLock> locks = MemoryLockUtils.getPathLocks(path, res.startsWith("R"));
+ locks.forEach(lock -> {
+ graph.setThread(threadId, lock);
+ lock.lock();
+ graph.resetThread(threadId);
+ holdLocks.add(lock);
+ });
+ } catch (Exception ignored) {
+ holdLocks.forEach(TransactionLock::unlock);
+ break;
+ }
+ }
+ });
+ t.start();
+ this.threads.add(t);
+ }
+ }
+
+ public void addLock(MemoryLockGraph graph, String... resourceToLock) throws InterruptedException {
+ for (int i = 0; i < resourceToLock.length; i++) {
+ for (String res : resourceToLock[i].split(",")) {
+ this.lockQueues.get(i).add(res);
+ }
+ }
+ // wait thread to take
+ for (int i = 0; i < threads.size(); i++) {
+ int finalI = i;
+ await().atMost(1, TimeUnit.SECONDS).until(() -> lockQueues.get(finalI).isEmpty()
+ || graph.isThreadWaitForLock(threads.get(finalI).getId()));
+ }
+ }
+
+ public void interruptAllThread() {
+ this.threads.forEach(Thread::interrupt);
+ }
+
+ public List<Long> getThreadIds() {
+ return this.threads.stream().map(Thread::getId).collect(Collectors.toList());
+ }
+ }
+}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java
new file mode 100644
index 0000000000..84f54c33bd
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/lock/rule/LockEliminationRuleExecTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.persistence.lock.rule;
+
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.MODEL;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.TABLE;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.TransactionLock;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class LockEliminationRuleExecTest extends NLocalFileMetadataTestCase {
+
+ @BeforeEach
+ public void setup() throws Exception {
+ createTestMetadata();
+ }
+
+ @AfterEach
+ public void teardown() {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ void testProjectRule() {
+ // ProjectLock ModuleLock PathLock
+ // Source: default TABLE table1,table2
+ // Expected: default NULL NULL
+
+ String project = "default";
+ {
+ List<TransactionLock> projectLock = MemoryLockUtils.getProjectLock(project);
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertFalse(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(project, lockInfo.getProjectLocks().get(0).transactionUnit());
+ }
+ {
+ List<TransactionLock> projectLock = MemoryLockUtils.getProjectLock(project);
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, TABLE);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertFalse(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(project, lockInfo.getProjectLocks().get(0).transactionUnit());
+ }
+ {
+ List<TransactionLock> projectLock = MemoryLockUtils.getProjectLock(project);
+ List<TransactionLock> moduleLocks = Collections.emptyList();
+ List<TransactionLock> pathLocks = Collections.emptyList();
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertFalse(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(project, lockInfo.getProjectLocks().get(0).transactionUnit());
+ }
+ }
+
+ @Test
+ void testModuleRule() {
+ // ProjectLock ModuleLock PathLock
+ // Source: NULL TABLE table1,table2
+ // Expected: NULL TABLE NULL
+ String project = "default";
+
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, TABLE);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(TABLE, lockInfo.getModuleLocks().get(0).moduleEnum());
+ }
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getPathLocks().isEmpty());
+ Assertions.assertEquals(MODEL, lockInfo.getModuleLocks().get(0).moduleEnum());
+ Assertions.assertEquals(2, lockInfo.getPathLocks().size());
+ }
+ }
+
+ @Test
+ void testPathRule() {
+ // ProjectLock ModuleLock PathLock
+ // Source: NULL MODEL table1,table2
+ // Expected: NULL MODEL,TABLE NULL
+ String project = "default";
+ KylinConfig.getInstanceFromEnv().setProperty("kylin.env.lock-remove-max-size", "1");
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils.getPathLocks(
+ Lists.newArrayList("/default/table/db.table1.json", "/default/table/db.table2.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertTrue(lockInfo.getPathLocks().isEmpty());
+
+ Assertions.assertEquals(2, lockInfo.getModuleLocks().size());
+ }
+ {
+ List<TransactionLock> projectLock = Collections.emptyList();
+ List<TransactionLock> moduleLocks = MemoryLockUtils.getModuleLocks(project, MODEL);
+ List<TransactionLock> pathLocks = MemoryLockUtils
+ .getPathLocks(Lists.newArrayList("/default/table/db.table1.json"));
+ LockInfo lockInfo = LockEliminationRuleExec.getInstance().execute(project, projectLock, moduleLocks,
+ pathLocks);
+ Assertions.assertTrue(lockInfo.getProjectLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getModuleLocks().isEmpty());
+ Assertions.assertFalse(lockInfo.getPathLocks().isEmpty());
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java
index deec6c65a1..9626449610 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStoreTest.java
@@ -43,11 +43,13 @@ import org.apache.kylin.common.persistence.StringEntity;
import org.apache.kylin.common.persistence.UnitMessages;
import org.apache.kylin.common.persistence.event.Event;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.jdbc.AuditLogRowMapper;
import org.apache.kylin.common.persistence.transaction.AuditLogReplayWorker;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.junit.JdbcInfo;
import org.apache.kylin.junit.annotation.JdbcMetadataInfo;
import org.apache.kylin.junit.annotation.MetadataInfo;
@@ -62,7 +64,6 @@ import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.util.ReflectionTestUtils;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -78,6 +79,10 @@ class JdbcAuditLogStoreTest {
void testUpdateResourceWithLog(JdbcInfo info) throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/abc", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc2", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc3", null, false);
+ MemoryLockUtils.lockAndRecord("/p1/abc4", null, false);
store.checkAndPutResource("/p1/abc", ByteSource.wrap("abc".getBytes(charset)), -1);
store.checkAndPutResource("/p1/abc2", ByteSource.wrap("abc".getBytes(charset)), -1);
store.checkAndPutResource("/p1/abc3", ByteSource.wrap("abc".getBytes(charset)), -1);
@@ -476,6 +481,7 @@ class JdbcAuditLogStoreTest {
public void testGet() throws IOException {
UnitOfWork.doInTransactionWithRetry(() -> {
val store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/p1/123", null, false);
store.checkAndPutResource("/p1/123", ByteSource.wrap("123".getBytes(charset)), -1);
return 0;
}, "p1");
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionLockTest.java b/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionProjectLockTest.java
similarity index 89%
rename from src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionLockTest.java
rename to src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionProjectLockTest.java
index 06c35d8113..ce86a92486 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionLockTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/persistence/transaction/TransactionProjectLockTest.java
@@ -19,23 +19,23 @@ package org.apache.kylin.common.persistence.transaction;
import java.util.concurrent.TimeUnit;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.assertj.core.util.Lists;
import org.awaitility.Awaitility;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class TransactionLockTest {
+@MetadataInfo(onlyProps = true)
+class TransactionProjectLockTest {
@Test
- public void testStravition() {
+ void testStravition() {
val threads = Lists.<LogicThread> newArrayList();
for (int i = 0; i < 10; i++) {
@@ -51,7 +51,6 @@ public class TransactionLockTest {
long minCount = threads.stream().mapToLong(LogicThread::getCount).min().getAsLong();
Assert.assertTrue(maxCount - minCount <= 1);
- TransactionLock.projectLocks = Maps.newConcurrentMap();
for (LogicThread thread : threads) {
thread.interrupt();
}
@@ -69,12 +68,12 @@ public class TransactionLockTest {
@Override
public void run() {
super.run();
- val tLock = TransactionLock.getLock("p0", isReader);
+ val tLock = TransactionManagerInstance.INSTANCE.getLock("p0", isReader);
while (count < 20) {
tLock.lock();
log.debug("lock by {} {}, {}", isReader ? "reader" : "writer", Thread.currentThread().getName(), count);
try {
- Thread.sleep(100);
+ TimeUnit.MILLISECONDS.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
index 0f58537aa7..5fc9a48e00 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
@@ -27,6 +27,7 @@ import java.util.Objects;
import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -104,8 +105,8 @@ public class SegmentUtil {
protected static <T extends ISegment> boolean anyIndexJobRunning(T segment) {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
NExecutableManager execManager = NExecutableManager.getInstance(kylinConfig, segment.getModel().getProject());
- val executables = execManager.listExecByJobTypeAndStatus(ExecutableState::isRunning, INDEX_BUILD,
- SUB_PARTITION_BUILD);
+ val executables = execManager.listPartialExec(path -> StringUtils.endsWith(path, segment.getModel().getId()),
+ ExecutableState::isRunning, INDEX_BUILD, SUB_PARTITION_BUILD);
return executables.stream().anyMatch(task -> task.getSegmentIds().contains(segment.getId()));
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java
index 72fcec4ccb..991b0ec504 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatistics.java
@@ -20,8 +20,11 @@ package org.apache.kylin.job.dao;
import java.util.Map;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.MetadataConstants;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -74,4 +77,9 @@ public class JobStatistics extends JobStatisticsBasic {
jobStatisticsByModels.put(model, jobStatisticsByModel);
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + getProject() + ResourceStore.JOB_STATISTICS + "/" + date + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java
index c05743c3cf..c1b6f7902d 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsBasic.java
@@ -20,6 +20,7 @@ package org.apache.kylin.job.dao;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
@@ -39,15 +40,22 @@ public class JobStatisticsBasic extends RootPersistentEntity {
@JsonProperty("total_byte_size")
private long totalByteSize;
+ @JsonIgnore
+ private String project;
+
public void update(long duration, long byteSize, int deltaCount) {
this.count += deltaCount;
this.totalDuration += duration;
this.totalByteSize += byteSize;
}
- public JobStatisticsBasic(long totalDuration, long totalByteSize) {
- this.count = 1;
+ public JobStatisticsBasic(int count, long totalDuration, long totalByteSize) {
+ this.count = count;
this.totalDuration = totalDuration;
this.totalByteSize = totalByteSize;
}
+
+ public JobStatisticsBasic(long totalDuration, long totalByteSize) {
+ this(1, totalDuration, totalByteSize);
+ }
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java
index 91acd80627..65d98d18c1 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobStatisticsManager.java
@@ -35,12 +35,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
public class JobStatisticsManager {
public static JobStatisticsManager getInstance(KylinConfig config, String project) {
@@ -86,28 +85,37 @@ public class JobStatisticsManager {
return crud.listAll();
}
+ public JobStatistics copyForWrite(JobStatistics jobStatistics) {
+ if (jobStatistics.getProject() == null) {
+ jobStatistics.setProject(project);
+ }
+ return crud.copyForWrite(jobStatistics);
+ }
+
public JobStatistics updateStatistics(long date, String model, long duration, long byteSize, int deltaCount) {
JobStatistics jobStatistics = crud.get(String.valueOf(date));
- JobStatistics jobStatisticsToUpdate;
if (jobStatistics == null) {
- jobStatisticsToUpdate = new JobStatistics(date, model, duration, byteSize);
- return crud.save(jobStatisticsToUpdate);
+ jobStatistics = copyForWrite(new JobStatistics(date, model, duration, byteSize));
+ // double check whether jobStatistics already exists.
+ if (jobStatistics.getMvcc() == -1) {
+ return crud.save(jobStatistics);
+ }
}
- jobStatisticsToUpdate = crud.copyForWrite(jobStatistics);
+ JobStatistics jobStatisticsToUpdate = copyForWrite(jobStatistics);
jobStatisticsToUpdate.update(model, duration, byteSize, deltaCount);
return crud.save(jobStatisticsToUpdate);
}
public JobStatistics updateStatistics(long date, long duration, long byteSize, int deltaCount) {
JobStatistics jobStatistics = crud.get(String.valueOf(date));
- JobStatistics jobStatisticsToUpdate;
if (jobStatistics == null) {
- jobStatisticsToUpdate = new JobStatistics(date, duration, byteSize);
- return crud.save(jobStatisticsToUpdate);
+ // double check whether jobStatistics already exists.
+ jobStatistics = copyForWrite(new JobStatistics(date, duration, byteSize));
+ return crud.save(jobStatistics);
}
- jobStatisticsToUpdate = crud.copyForWrite(jobStatistics);
+ JobStatistics jobStatisticsToUpdate = copyForWrite(jobStatistics);
jobStatisticsToUpdate.update(duration, byteSize, deltaCount);
return crud.save(jobStatisticsToUpdate);
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java
index 130c5539a0..bcc4a497f2 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/NExecutableDao.java
@@ -26,25 +26,20 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import lombok.val;
/**
*/
public class NExecutableDao {
- private static final Serializer<ExecutablePO> JOB_SERIALIZER = new JsonSerializer<>(ExecutablePO.class);
private static final Logger logger = LoggerFactory.getLogger(NExecutableDao.class);
public static NExecutableDao getInstance(KylinConfig config, String project) {
@@ -103,6 +98,13 @@ public class NExecutableDao {
return crud.get(uuid);
}
+ public ExecutablePO copyForWrite(ExecutablePO po) {
+ if (po.getProject() == null) {
+ po.setProject(project);
+ }
+ return crud.copyForWrite(po);
+ }
+
public ExecutablePO addJob(ExecutablePO job) {
if (getJobByUuid(job.getUuid()) != null) {
throw new IllegalArgumentException("job id:" + job.getUuid() + " already exists");
@@ -124,7 +126,7 @@ public class NExecutableDao {
public void updateJob(String uuid, Predicate<ExecutablePO> updater) {
val job = getJobByUuid(uuid);
Preconditions.checkNotNull(job);
- val copyForWrite = JsonUtil.copyBySerialization(job, JOB_SERIALIZER, null);
+ val copyForWrite = copyForWrite(job);
if (updater.test(copyForWrite)) {
crud.save(copyForWrite);
}
@@ -137,7 +139,7 @@ public class NExecutableDao {
} else {
ExecutablePO executablePOFromCache = getJobByUuid(uuid);
Preconditions.checkNotNull(executablePOFromCache);
- val copyForWrite = JsonUtil.copyBySerialization(executablePOFromCache, JOB_SERIALIZER, null);
+ val copyForWrite = copyForWrite(executablePOFromCache);
updating.put(uuid, copyForWrite);
executablePO = copyForWrite;
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
index 5eca8383e9..83ce486279 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
@@ -199,10 +199,11 @@ public class NExecutableManager {
}
public void addJob(ExecutablePO executablePO) {
- addJobOutput(executablePO);
- executableDao.addJob(executablePO);
+ ExecutablePO copy = executableDao.copyForWrite(executablePO);
+ addJobOutput(copy);
+ executableDao.addJob(copy);
- String jobType = executablePO.getJobType() == null ? "" : executablePO.getJobType().name();
+ String jobType = copy.getJobType() == null ? "" : copy.getJobType().name();
// dispatch job-created message out
if (KylinConfig.getInstanceFromEnv().isUTEnv()) {
EventBusFactory.getInstance().postAsync(new JobReadyNotifier(project));
@@ -1410,7 +1411,11 @@ public class NExecutableManager {
val thread = scheduler.getContext().getRunningJobThread(executable);
if (thread != null) {
logger.info("Interrupt Job [{}] thread and remove in ExecutableContext", executable.getDisplayName());
- thread.interrupt();
+ // workaround for https://olapio.atlassian.net/browse/KE-41836
+ if (Arrays.stream(thread.getStackTrace())
+ .noneMatch(stack -> stack.getClassName().contains("FetcherRunner"))) {
+ thread.interrupt();
+ }
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java
index 15fb55a597..52a8db9fee 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/TempStatementManager.java
@@ -74,6 +74,10 @@ public class TempStatementManager {
public TempStatementEntity getTempStatEntity(String sessionId, String statementId) {
return crud.get(TempStatementEntity.resourceName(sessionId, statementId));
}
+
+ private TempStatementEntity copyForWrite(TempStatementEntity entity) {
+ return crud.copyForWrite(entity);
+ }
public void updateTempStatement(String statementId, String statement) throws IOException {
updateTempStatement(TempStatementEntity.DEFAULT_SESSION_ID, statementId, statement);
@@ -86,13 +90,13 @@ public class TempStatementManager {
}
private TempStatementEntity prepareToOverwrite(TempStatementEntity entity, TempStatementEntity origin) {
+ TempStatementEntity copy = copyForWrite(entity);
if (origin == null) {
// create
- entity.updateRandomUuid();
+ copy.updateRandomUuid();
} else {
// update
- entity.setUuid(origin.getUuid());
- entity.setLastModified(origin.getLastModified());
+ copy.setStatement(entity.getStatement());
}
return entity;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java
index d15b9f741d..0b51b10574 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCR.java
@@ -38,6 +38,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -57,9 +58,13 @@ public class AclTCR extends RootPersistentEntity {
//wrap read only aclTCR
private String resourceName;
+ private String project;
+ private boolean principal;
- public void init(String resourceName) {
+ public void init(String resourceName, String project, boolean principal) {
this.resourceName = resourceName;
+ this.project = project;
+ this.principal = principal;
}
@JsonProperty
@@ -70,6 +75,16 @@ public class AclTCR extends RootPersistentEntity {
return resourceName;
}
+ @Override
+ public String getResourcePath() {
+ return "/" + project + "/acl/" + (principal ? "user/" : "group/") + resourceName()
+ + MetadataConstants.FILE_SURFIX;
+ }
+
+ public boolean isPrincipal() {
+ return principal;
+ }
+
public Table getTable() {
return table;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
index 044bdfa7ff..7280ae81ec 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/acl/AclTCRManager.java
@@ -89,7 +89,7 @@ public class AclTCRManager {
AclTCR.class) {
@Override
protected AclTCR initEntityAfterReload(AclTCR acl, String resourceName) {
- acl.init(resourceName);
+ acl.init(resourceName, project, true);
return acl;
}
};
@@ -99,7 +99,7 @@ public class AclTCRManager {
AclTCR.class) {
@Override
protected AclTCR initEntityAfterReload(AclTCR acl, String resourceName) {
- acl.init(resourceName);
+ acl.init(resourceName, project, false);
return acl;
}
};
@@ -134,21 +134,18 @@ public class AclTCRManager {
}
public void updateAclTCR(AclTCR updateTo, String sid, boolean principal) {
- updateTo.init(sid);
+ updateTo.init(sid, project, principal);
if (principal) {
- doUpdate(updateTo, sid, userCrud);
+ doUpdate(updateTo, userCrud);
} else {
- doUpdate(updateTo, sid, groupCrud);
+ doUpdate(updateTo, groupCrud);
}
}
- private void doUpdate(AclTCR updateTo, String sid, CachedCrudAssist<AclTCR> crud) {
- AclTCR copied;
- AclTCR origin = crud.get(sid);
- if (Objects.isNull(origin)) {
- copied = updateTo;
- } else {
- copied = crud.copyForWrite(origin);
+ private void doUpdate(AclTCR updateTo, CachedCrudAssist<AclTCR> crud) {
+ AclTCR cached = crud.get(updateTo.resourceName());
+ AclTCR copied = crud.copyForWrite(cached == null ? updateTo : cached);
+ if (copied.getMvcc() != -1) {
copied.setTable(updateTo.getTable());
}
crud.save(copied);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java
index 47b5f076d9..a5a114bc2c 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CacheReloadChecker.java
@@ -20,7 +20,7 @@ package org.apache.kylin.metadata.cachesync;
import org.apache.kylin.common.persistence.MissingRootPersistentEntity;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-
+import org.apache.kylin.common.persistence.metadata.FileMetadataStore;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.AllArgsConstructor;
@@ -53,6 +53,10 @@ public class CacheReloadChecker<T extends RootPersistentEntity> {
if (raw.getMvcc() != entity.getMvcc()) {
return true;
}
+ if ((store.getMetadataStore() instanceof FileMetadataStore)
+ && (raw.getTimestamp() != entity.getLastModified())) {
+ return true;
+ }
Preconditions.checkState(!(entity instanceof MissingRootPersistentEntity));
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
index 3793ca7ff2..fdc4851213 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
@@ -21,24 +21,27 @@ package org.apache.kylin.metadata.cachesync;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.ThreadUtil;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.util.BrokenEntityProxy;
-
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.cache.Cache;
import org.apache.kylin.guava30.shaded.common.cache.CustomKeyEquivalenceCacheBuilder;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.util.BrokenEntityProxy;
import lombok.AccessLevel;
import lombok.Getter;
@@ -88,9 +91,47 @@ public abstract class CachedCrudAssist<T extends RootPersistentEntity> {
this.checkCopyOnWrite = check;
}
- // Make copy of an entity such that update can apply on the copy.
- // Note cached and shared object MUST NOT be updated directly.
+ // If this is a new entity or the reload entity is broken, return a copy of this entity.
+ // If the reload entity is null, just return null.
+ // Otherwise return a copy of the reload entity.
public T copyForWrite(T entity) {
+ if (entity == null) {
+ return null;
+ }
+ String resourcePath = resourcePath(entity.resourceName());
+ return MemoryLockUtils.doWithLock(entity, resourcePath, false, store, () -> {
+ if (entity.getMvcc() == -1) {
+ return copyIfCachedAndShared(entity);
+ }
+ T reloadedEntity = get(entity.resourceName());
+ if (reloadedEntity == null) {
+ return null;
+ } else if (reloadedEntity.isBroken()) {
+ return copyIfCachedAndShared(entity);
+ } else {
+ return copyIfCachedAndShared(reloadedEntity);
+ }
+ });
+ }
+
+ // Used by managers which doesn't have cache.
+ public static <T extends RootPersistentEntity> T copyForWrite(T entity, Serializer<T> serializer,
+ @Nullable BiConsumer<T, String> initEntity, ResourceStore store) {
+ String resourcePath = entity.getResourcePath();
+ return MemoryLockUtils.doWithLock(entity, resourcePath, false, store, () -> {
+ T reloadedEntity = store.getResource(entity.getResourcePath(), serializer);
+ if (reloadedEntity == null || reloadedEntity.isBroken()) {
+ return JsonUtil.copyForWrite(entity, serializer, initEntity);
+ } else {
+ if (initEntity != null) {
+ initEntity.accept(reloadedEntity, reloadedEntity.resourceName());
+ }
+ return JsonUtil.copyForWrite(reloadedEntity, serializer, initEntity);
+ }
+ });
+ }
+
+ public T copyIfCachedAndShared(T entity) {
return JsonUtil.copyForWrite(entity, serializer, this::initEntityAfterReload);
}
@@ -98,7 +139,7 @@ public abstract class CachedCrudAssist<T extends RootPersistentEntity> {
return JsonUtil.copyBySerialization(entity, serializer, this::initEntityAfterReload);
}
- String resourcePath(String resourceName) {
+ public String resourcePath(String resourceName) {
Preconditions.checkArgument(StringUtils.isNotEmpty(resourceName),
"The resource name \"{}\" cannot contain white character", resourceName);
return resRootPath + "/" + resourceName + resPathSuffix;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
index ee80805d1a..83edf8385b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
@@ -365,6 +365,11 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
.append("/").append(name).append(MetadataConstants.FILE_SURFIX).toString();
}
+ @Override
+ public List<String> getLockPaths(String ignored) {
+ return getModel().getLockPaths();
+ }
+
public String getProject() {
return project;
}
@@ -583,6 +588,9 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
public void setRuleBasedIndex(RuleBasedIndex ruleBasedIndex, Set<LayoutEntity> reloadLayouts, boolean reuseStartId,
boolean markToBeDeleted, boolean restoreDeletedIndex) {
+ if (ruleBasedIndex == null) {
+ return;
+ }
checkIsNotCachedAndShared();
ruleBasedIndex.adjustMeasures();
if (CollectionUtils.isEmpty(ruleBasedIndex.getMeasures())) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
index c9d1ea969e..9f86786f11 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManager.java
@@ -26,23 +26,22 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.ManagementType;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.model.ManagementType;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.var;
@@ -113,61 +112,74 @@ public class NDataLoadingRangeManager {
}
public NDataLoadingRange createDataLoadingRange(NDataLoadingRange dataLoadingRange) {
- checkNDataLoadingRangeIdentify(dataLoadingRange);
- checkNDataLoadingRangeExist(dataLoadingRange);
+ NDataLoadingRange copy = copyForWrite(dataLoadingRange);
+ checkNDataLoadingRangeIdentify(copy);
+ checkNDataLoadingRangeExist(copy);
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(config, project);
- String tableName = dataLoadingRange.getTableName();
+ String tableName = copy.getTableName();
TableDesc tableDesc = tableMetadataManager.getTableDesc(tableName);
if (tableDesc == null) {
- throw new IllegalArgumentException(DATA_LOADING_RANGE + dataLoadingRange.resourceName() + "' 's table "
- + tableName + " does not exists");
+ throw new IllegalArgumentException(
+ DATA_LOADING_RANGE + copy.resourceName() + "' 's table " + tableName + " does not exists");
}
- String columnName = dataLoadingRange.getColumnName();
+ String columnName = copy.getColumnName();
ColumnDesc columnDesc = tableDesc.findColumnByName(columnName);
if (columnDesc == null) {
- throw new IllegalArgumentException(DATA_LOADING_RANGE + dataLoadingRange.resourceName() + "' 's column "
- + columnName + " does not exists");
+ throw new IllegalArgumentException(
+ DATA_LOADING_RANGE + copy.resourceName() + "' 's column " + columnName + " does not exists");
}
String columnType = columnDesc.getDatatype();
DataType dataType = DataType.getType(columnType);
if (dataType == null || !dataType.isLegalPartitionColumnType()) {
- throw new IllegalArgumentException(DATA_LOADING_RANGE + dataLoadingRange.resourceName() + "' 's column "
- + columnName + " 's dataType does not support partition column");
+ throw new IllegalArgumentException(DATA_LOADING_RANGE + copy.resourceName() + "' 's column " + columnName
+ + " 's dataType does not support partition column");
}
- return crud.save(dataLoadingRange);
+ return crud.save(copy);
}
public NDataLoadingRange appendSegmentRange(NDataLoadingRange dataLoadingRange, SegmentRange segmentRange) {
- NDataLoadingRange copyForWrite = copyForWrite(dataLoadingRange);
- val coveredRange = copyForWrite.getCoveredRange();
- if (coveredRange == null) {
- copyForWrite.setCoveredRange(segmentRange);
- } else {
- if (coveredRange.connects(segmentRange)) {
- copyForWrite.setCoveredRange(coveredRange.coverWith(segmentRange));
- } else if (segmentRange.connects(coveredRange)) {
- copyForWrite.setCoveredRange(segmentRange.coverWith(coveredRange));
+ return updateDataLoadingRange(dataLoadingRange.getTableName(), copyForWrite -> {
+ val coveredRange = copyForWrite.getCoveredRange();
+ if (coveredRange == null) {
+ copyForWrite.setCoveredRange(segmentRange);
} else {
- throw new IllegalArgumentException("NDataLoadingRange appendSegmentRange " + segmentRange
- + " has overlaps/gap with existing segmentRanges " + copyForWrite.getCoveredRange());
+ if (coveredRange.connects(segmentRange)) {
+ copyForWrite.setCoveredRange(coveredRange.coverWith(segmentRange));
+ } else if (segmentRange.connects(coveredRange)) {
+ copyForWrite.setCoveredRange(segmentRange.coverWith(coveredRange));
+ } else {
+ throw new IllegalArgumentException("NDataLoadingRange appendSegmentRange " + segmentRange
+ + " has overlaps/gap with existing segmentRanges " + copyForWrite.getCoveredRange());
+ }
}
- }
- return updateDataLoadingRange(copyForWrite);
+ return true;
+ });
}
+ /**
+ * @deprecated Use updateDataLoadingRange(String tableName, DataLoadingRangeUpdater updater) instead.
+ */
+ @Deprecated
public NDataLoadingRange updateDataLoadingRange(NDataLoadingRange dataLoadingRange) {
- if (getStore().getConfig().isCheckCopyOnWrite()) {
- if (dataLoadingRange.isCachedAndShared())
- throw new IllegalStateException();
- }
- checkNDataLoadingRangeIdentify(dataLoadingRange);
- checkNDataLoadingRangeNotExist(dataLoadingRange);
-
- return crud.save(dataLoadingRange);
+ return updateDataLoadingRange(dataLoadingRange.getTableName(), copyForWrite -> {
+ dataLoadingRange.copyPropertiesTo(copyForWrite);
+ return true;
+ });
}
+ public NDataLoadingRange updateDataLoadingRange(String tableName, DataLoadingRangeUpdater updater) {
+ NDataLoadingRange cached = getDataLoadingRange(tableName);
+ NDataLoadingRange copy = copyForWrite(cached);
+ if (!updater.modify(copy)) {
+ return null;
+ }
+ checkNDataLoadingRangeIdentify(copy);
+ checkNDataLoadingRangeNotExist(copy);
+ return crud.save(copy);
+ }
+
public NDataLoadingRange copyForWrite(NDataLoadingRange dataLoadingRange) {
return crud.copyForWrite(dataLoadingRange);
}
@@ -251,16 +263,21 @@ public class NDataLoadingRangeManager {
if (loadingRange == null) {
return;
} else {
- val copy = copyForWrite(loadingRange);
- val coveredRange = copy.getCoveredRange();
- if (coveredRange == null) {
- return;
- }
- if (lastSegment.getSegRange().overlaps(coveredRange)) {
- copy.setCoveredRange(lastSegment.getSegRange().getEndDeviation(coveredRange));
- updateDataLoadingRange(copy);
- }
-
+ updateDataLoadingRange(loadingRange.getTableName(), copyForWrite -> {
+ val coveredRange = copyForWrite.getCoveredRange();
+ if (coveredRange == null) {
+ return false;
+ }
+ if (lastSegment.getSegRange().overlaps(coveredRange)) {
+ copyForWrite.setCoveredRange(lastSegment.getSegRange().getEndDeviation(coveredRange));
+ return true;
+ }
+ return false;
+ });
}
}
+
+ public interface DataLoadingRangeUpdater {
+ boolean modify(NDataLoadingRange copyForWrite);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java
index 86676b7f8b..982bb5d7ff 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManager.java
@@ -31,13 +31,13 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import lombok.val;
/**
@@ -65,6 +65,7 @@ class NDataSegDetailsManager {
private KylinConfig kylinConfig;
private String project;
+ private ResourceStore resourceStore;
private NDataSegDetailsManager(KylinConfig config, String project) {
if (!UnitOfWork.isAlreadyInTransaction())
@@ -72,6 +73,7 @@ class NDataSegDetailsManager {
System.identityHashCode(config), project);
this.kylinConfig = config;
this.project = project;
+ this.resourceStore = ResourceStore.getKylinMetaStore(this.kylinConfig);
}
public KylinConfig getConfig() {
@@ -79,9 +81,18 @@ class NDataSegDetailsManager {
}
private ResourceStore getStore() {
- return ResourceStore.getKylinMetaStore(this.kylinConfig);
+ return resourceStore;
}
+ private NDataSegDetails initEntity(NDataSegDetails dataSegDetails, String resourceName) {
+ dataSegDetails.setProject(project);
+ return dataSegDetails;
+ }
+
+ public NDataSegDetails copyForWrite(NDataSegDetails details) {
+ return CachedCrudAssist.copyForWrite(details, DATA_SEG_LAYOUT_INSTANCES_SERIALIZER, this::initEntity, resourceStore);
+ }
+
NDataSegDetails getForSegment(NDataflow df, String segId) {
NDataSegDetails instances = getStore().getResource(getResourcePathForSegment(df.getUuid(), segId),
DATA_SEG_LAYOUT_INSTANCES_SERIALIZER);
@@ -127,20 +138,21 @@ class NDataSegDetailsManager {
NDataSegDetails details = getForSegment(df, segId);
if (details == null)
details = NDataSegDetails.newSegDetails(df, segId);
+ NDataSegDetails copy = copyForWrite(details);
if (toUpsert.containsKey(segId)) {
for (NDataLayout c : toUpsert.get(segId)) {
- c.setSegDetails(details);
- details.addLayout(c);
+ c.setSegDetails(copy);
+ copy.addLayout(c);
}
}
if (toRemove.containsKey(segId)) {
for (NDataLayout c : toRemove.get(segId)) {
- details.removeLayout(c);
+ copy.removeLayout(c);
}
}
- upsertForSegmentQuietly(details);
+ upsertForSegmentQuietly(copy);
}
if (update.getToRemoveSegs() != null) {
@@ -163,13 +175,24 @@ class NDataSegDetailsManager {
}
}
- NDataSegDetails upsertForSegment(NDataSegDetails details) {
+ private NDataSegDetails upsertForSegment(NDataSegDetails details) {
Preconditions.checkNotNull(details, "NDataSegDetails cannot be null.");
getStore().checkAndPutResource(details.getResourcePath(), details, DATA_SEG_LAYOUT_INSTANCES_SERIALIZER);
return details;
}
+ public NDataSegDetails updateDetails(NDataSegment seg, NDataSegDetailsUpdater updater) {
+ NDataSegDetails details = copyForWrite(getForSegment(seg));
+ updater.modify(details);
+ upsertForSegment(details);
+ return getForSegment(seg);
+ }
+
+ interface NDataSegDetailsUpdater {
+ void modify(NDataSegDetails copyForWrite);
+ }
+
private void removeForSegmentQuietly(NDataflow df, String segId) {
try {
removeForSegment(df, segId);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
index 022d7a39c4..ce2d5c817e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowManager.java
@@ -31,6 +31,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -48,6 +49,9 @@ import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.model.ManagementType;
import org.apache.kylin.metadata.model.NDataModel;
@@ -66,10 +70,6 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
import lombok.val;
import lombok.var;
@@ -277,14 +277,15 @@ public class NDataflowManager implements IRealizationProvider {
public NDataflow createDataflow(IndexPlan plan, String owner, RealizationStatusEnum realizationStatusEnum) {
NDataflow df = NDataflow.create(plan, realizationStatusEnum);
df.initAfterReload((KylinConfigExt) plan.getConfig(), project);
+ NDataflow copy = copyForWrite(df);
// save dataflow
- df.getSegments().validate();
- crud.save(df);
+ copy.getSegments().validate();
+ crud.save(copy);
- fillDf(df);
+ fillDf(copy);
- return df;
+ return copy;
}
public void fillDf(NDataflow df) {
@@ -339,22 +340,22 @@ public class NDataflowManager implements IRealizationProvider {
}
public NDataSegment appendPartitions(String dfId, String segId, List<String[]> partitionValues) {
- val copy = copy(getDataflow(dfId));
- val segmentCopy = copy.getSegment(segId);
- partitionValues.forEach(partitionValue -> {
- if (copy.getSegment(segmentCopy.getId()).isPartitionOverlap(partitionValue)) {
- throw new IllegalArgumentException(
- String.format(Locale.ROOT, "Duplicate partition value [%s] found in segment [%s]",
- Arrays.toString(partitionValue), segmentCopy.getId()));
- }
- });
- val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- Set<Long> addPartitions = modelManager.addPartitionsIfAbsent(copy.getModel(), partitionValues);
- addPartitions.forEach(partition -> {
- segmentCopy.getMultiPartitions().add(new SegmentPartition(partition));
+ updateDataflow(dfId, copyForWrite -> {
+ val segmentCopy = copyForWrite.getSegment(segId);
+ partitionValues.forEach(partitionValue -> {
+ if (copyForWrite.getSegment(segmentCopy.getId()).isPartitionOverlap(partitionValue)) {
+ throw new IllegalArgumentException(
+ String.format(Locale.ROOT, "Duplicate partition value [%s] found in segment [%s]",
+ Arrays.toString(partitionValue), segmentCopy.getId()));
+ }
+ });
+ val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ Set<Long> addPartitions = modelManager.addPartitionsIfAbsent(copyForWrite.getModel(), partitionValues);
+ addPartitions.forEach(partition -> {
+ segmentCopy.getMultiPartitions().add(new SegmentPartition(partition));
+ });
});
- crud.save(copy);
- return segmentCopy;
+ return copy(getDataflow(dfId)).getSegment(segId);
}
public NDataSegment appendSegmentForStreaming(NDataflow df, SegmentRange segRange) {
@@ -581,6 +582,10 @@ public class NDataflowManager implements IRealizationProvider {
return toRemoveSegs;
}
+ public NDataflow copyForWrite(NDataflow df) {
+ return crud.copyForWrite(df);
+ }
+
public NDataflow copy(NDataflow df) {
return crud.copyBySerialization(df);
}
@@ -632,7 +637,7 @@ public class NDataflowManager implements IRealizationProvider {
*/
public NDataflow updateDataflow(String dfId, NDataflowUpdater updater) {
NDataflow cached = getDataflow(dfId);
- NDataflow copy = copy(cached);
+ NDataflow copy = copyForWrite(cached);
updater.modify(copy);
Set<String> copySegIdSet = copy.getSegments().stream().map(NDataSegment::getId).collect(Collectors.toSet());
@@ -661,15 +666,23 @@ public class NDataflowManager implements IRealizationProvider {
return getDataflow(modelId).getLastBuildTime();
}
- public void updateDataflowDetailsLayouts(final NDataSegment seg, final List<NDataLayout> layouts) {
+ public void updateDataflowDetailsLayouts(final NDataSegment seg, final List<Long> toRemoveLayouts,
+ final List<Long> toAddLayouts) {
NDataSegDetailsManager segDetailsManager = NDataSegDetailsManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- NDataSegDetails details = segDetailsManager.getForSegment(seg);
- details.setLayouts(layouts);
- NDataSegDetailsManager.getInstance(KylinConfig.getInstanceFromEnv(), project).upsertForSegment(details);
+ segDetailsManager.updateDetails(seg, copyForWrite -> {
+ List<NDataLayout> layouts = new LinkedList<>(copyForWrite.getAllLayouts());
+ layouts.removeIf(layout -> toRemoveLayouts.contains(layout.getLayoutId()));
+ List<Long> existLayouts = layouts.stream().map(NDataLayout::getLayoutId).collect(Collectors.toList());
+ for (Long layoutId : toAddLayouts) {
+ if (!existLayouts.contains(layoutId)) {
+ layouts.add(NDataLayout.newDataLayout(copyForWrite, layoutId));
+ }
+ }
+ copyForWrite.setLayouts(layouts);
+ });
updateDataflow(seg.getDataflow().getId(),
copyForWrite -> updateSegmentStatus(copyForWrite.getSegment(seg.getId())));
-
}
public NDataflow updateDataflow(final NDataflowUpdate update) {
@@ -803,20 +816,20 @@ public class NDataflowManager implements IRealizationProvider {
}
public void removeSegmentPartition(String dfId, Set<Long> toBeDeletedPartIds, Set<String> segments) {
- val dfCopy = copy(getDataflow(dfId));
- val updateSegments = new Segments<NDataSegment>();
- if (CollectionUtils.isEmpty(segments)) {
- updateSegments.addAll(dfCopy.getSegments());
- } else {
- dfCopy.getSegments().forEach(segment -> {
- if (segments.contains(segment.getId())) {
- updateSegments.add(segment);
- }
- });
- }
- updateSegments.forEach(segment -> segment.getMultiPartitions()
- .removeIf(partition -> toBeDeletedPartIds.contains(partition.getPartitionId())));
- crud.save(dfCopy);
+ updateDataflow(dfId, copyForWrite -> {
+ val updateSegments = new Segments<NDataSegment>();
+ if (CollectionUtils.isEmpty(segments)) {
+ updateSegments.addAll(copyForWrite.getSegments());
+ } else {
+ copyForWrite.getSegments().forEach(segment -> {
+ if (segments.contains(segment.getId())) {
+ updateSegments.add(segment);
+ }
+ });
+ }
+ updateSegments.forEach(segment -> segment.getMultiPartitions()
+ .removeIf(partition -> toBeDeletedPartIds.contains(partition.getPartitionId())));
+ });
}
public void removeLayoutPartition(String dfId, Set<Long> toBeDeletedPartIds, Set<String> segments) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
index e5e605da6e..670e8f5af7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NIndexPlanManager.java
@@ -32,9 +32,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.metadata.cube.cuboid.CuboidScheduler;
import org.apache.kylin.metadata.cube.model.validation.NIndexPlanValidator;
import org.apache.kylin.metadata.model.NDataModel;
@@ -42,9 +44,6 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.val;
public class NIndexPlanManager {
@@ -94,6 +93,13 @@ public class NIndexPlanManager {
this.crud.setCheckCopyOnWrite(true);
}
+ public IndexPlan copyForWrite(IndexPlan plan) {
+ if (plan.getProject() == null) {
+ plan.setProject(project);
+ }
+ return crud.copyForWrite(plan);
+ }
+
public IndexPlan copy(IndexPlan plan) {
return crud.copyBySerialization(plan);
}
@@ -123,8 +129,6 @@ public class NIndexPlanManager {
public IndexPlan createIndexPlan(IndexPlan indexPlan) {
if (indexPlan.getUuid() == null)
throw new IllegalArgumentException();
- if (crud.contains(indexPlan.getUuid()))
- throw new IllegalArgumentException("IndexPlan '" + indexPlan.getUuid() + "' already exists");
try {
// init the cube plan if not yet
@@ -135,18 +139,22 @@ public class NIndexPlanManager {
indexPlan.addError(e.getMessage());
}
+ IndexPlan copy = copyForWrite(indexPlan);
+ if (crud.contains(copy.getUuid()))
+ throw new IllegalArgumentException("IndexPlan '" + indexPlan.getUuid() + "' already exists");
+
// Check base validation
- if (!indexPlan.getError().isEmpty()) {
- throw new IllegalArgumentException(indexPlan.getErrorMsg());
+ if (!copy.getError().isEmpty()) {
+ throw new IllegalArgumentException(copy.getErrorMsg());
}
// Semantic validation
NIndexPlanValidator validator = new NIndexPlanValidator();
- ValidateContext context = validator.validate(indexPlan);
+ ValidateContext context = validator.validate(copy);
if (!context.ifPass()) {
- throw new IllegalArgumentException(indexPlan.getErrorMsg());
+ throw new IllegalArgumentException(copy.getErrorMsg());
}
- return save(indexPlan);
+ return save(copy);
}
public interface NIndexPlanUpdater {
@@ -155,14 +163,18 @@ public class NIndexPlanManager {
public IndexPlan updateIndexPlan(String indexPlanId, NIndexPlanUpdater updater) {
IndexPlan cached = getIndexPlan(indexPlanId);
- IndexPlan copy = copy(cached);
+ IndexPlan copy = copyForWrite(cached);
updater.modify(copy);
- return updateIndexPlan(copy);
+ return innerUpdateIndexPlan(copy);
}
// use the NIndexPlanUpdater instead
@Deprecated
public IndexPlan updateIndexPlan(IndexPlan indexPlan) {
+ return updateIndexPlan(indexPlan.getUuid(), indexPlan::copyPropertiesTo);
+ }
+
+ private IndexPlan innerUpdateIndexPlan(IndexPlan indexPlan) {
if (indexPlan.isCachedAndShared())
throw new IllegalStateException();
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
index 1bbd33831f..4b08f4a5cc 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AbstractAsyncTask.java
@@ -18,8 +18,11 @@
package org.apache.kylin.metadata.favorite;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.MetadataConstants;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.EqualsAndHashCode;
@@ -36,7 +39,15 @@ public class AbstractAsyncTask extends RootPersistentEntity {
@JsonProperty("task_type")
private String taskType;
+ @JsonIgnore
+ private String project;
+
public AbstractAsyncTask(String taskType) {
this.taskType = taskType;
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + project + ResourceStore.ASYNC_TASK + "/" + uuid + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java
index 55eb565364..7560659ab8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncAccelerationTask.java
@@ -20,13 +20,14 @@ package org.apache.kylin.metadata.favorite;
import java.util.Map;
+import org.apache.kylin.common.annotation.Clarification;
+
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
-import org.apache.kylin.common.annotation.Clarification;
@Getter
@Setter
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java
index d2a4daa9b5..afdbe3eb61 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/AsyncTaskManager.java
@@ -33,6 +33,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import lombok.extern.slf4j.Slf4j;
@@ -47,11 +48,13 @@ public class AsyncTaskManager {
private final KylinConfig kylinConfig;
private final ResourceStore resourceStore;
private final String resourceRoot;
+ private final String project;
private AsyncTaskManager(KylinConfig kylinConfig, String project) {
if (!UnitOfWork.isAlreadyInTransaction())
log.info("Initializing AccelerateTagManager with KylinConfig Id: {} for project {}",
System.identityHashCode(kylinConfig), project);
+ this.project = project;
this.kylinConfig = kylinConfig;
resourceStore = ResourceStore.getKylinMetaStore(this.kylinConfig);
this.resourceRoot = "/" + project + ResourceStore.ASYNC_TASK;
@@ -76,6 +79,13 @@ public class AsyncTaskManager {
}
}
+ public AsyncAccelerationTask copyForWrite(AsyncAccelerationTask task) {
+ if (task.getProject() == null) {
+ task.setProject(project);
+ }
+ return CachedCrudAssist.copyForWrite(task, ASYNC_ACCELERATION_SERIALIZER, null, resourceStore);
+ }
+
public AbstractAsyncTask get(String taskType) {
List<AsyncAccelerationTask> asyncAccelerationTaskList = Lists.newArrayList();
if (taskType.equalsIgnoreCase(ASYNC_ACCELERATION_TASK)) {
@@ -90,11 +100,12 @@ public class AsyncTaskManager {
public static void resetAccelerationTagMap(String project) {
log.info("reset acceleration tag for project({})", project);
EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) getInstance(
- KylinConfig.getInstanceFromEnv(), project).get(ASYNC_ACCELERATION_TASK);
- asyncAcceleration.setAlreadyRunning(false);
- asyncAcceleration.setUserRefreshedTagMap(Maps.newHashMap());
- getInstance(KylinConfig.getInstanceFromEnv(), project).save(asyncAcceleration);
+ AsyncTaskManager manager = getInstance(KylinConfig.getInstanceFromEnv(), project);
+ AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) manager.get(ASYNC_ACCELERATION_TASK);
+ AsyncAccelerationTask copied = manager.copyForWrite(asyncAcceleration);
+ copied.setAlreadyRunning(false);
+ copied.setUserRefreshedTagMap(Maps.newHashMap());
+ getInstance(KylinConfig.getInstanceFromEnv(), project).save(copied);
return null;
}, project);
log.info("rest acceleration tag successfully for project({})", project);
@@ -114,10 +125,11 @@ public class AsyncTaskManager {
log.info("start to clean acceleration tag by user");
EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) getInstance(
- KylinConfig.getInstanceFromEnv(), project).get(ASYNC_ACCELERATION_TASK);
- asyncAcceleration.getUserRefreshedTagMap().put(userName, false);
- getInstance(KylinConfig.getInstanceFromEnv(), project).save(asyncAcceleration);
+ AsyncTaskManager manager = getInstance(KylinConfig.getInstanceFromEnv(), project);
+ AsyncAccelerationTask asyncAcceleration = (AsyncAccelerationTask) manager.get(ASYNC_ACCELERATION_TASK);
+ AsyncAccelerationTask copied = manager.copyForWrite(asyncAcceleration);
+ copied.getUserRefreshedTagMap().put(userName, false);
+ getInstance(KylinConfig.getInstanceFromEnv(), project).save(copied);
return null;
}, project);
log.info("clean acceleration tag successfully for project({}: by user {})", project, userName);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java
index 89a3368182..f806795547 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRule.java
@@ -23,14 +23,17 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.kylin.common.annotation.Clarification;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.RandomUtil;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.MetadataConstants;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -75,6 +78,9 @@ public class FavoriteRule extends RootPersistentEntity {
@JsonProperty("enabled")
private boolean enabled;
+ @JsonIgnore
+ private String project;
+
public static List<FavoriteRule> getAllDefaultRule() {
return FAVORITE_RULE_NAMES.stream().map(ruleName -> getDefaultRuleIfNull(null, ruleName))
.collect(Collectors.toList());
@@ -124,6 +130,11 @@ public class FavoriteRule extends RootPersistentEntity {
return null;
}
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + project + ResourceStore.QUERY_FILTER_RULE_RESOURCE_ROOT + "/" + resourceName() + MetadataConstants.FILE_SURFIX;
+ }
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
@NoArgsConstructor
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java
index ecbaa4bd80..fb685a2022 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/FavoriteRuleManager.java
@@ -29,14 +29,13 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.annotation.Clarification;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import lombok.val;
@Clarification(priority = Clarification.Priority.MAJOR, msg = "Enterprise")
@@ -75,6 +74,7 @@ public class FavoriteRuleManager {
this.crud = new CachedCrudAssist<FavoriteRule>(store, resourceRoot, FavoriteRule.class) {
@Override
protected FavoriteRule initEntityAfterReload(FavoriteRule entity, String resourceName) {
+ entity.setProject(project);
return entity;
}
};
@@ -112,6 +112,13 @@ public class FavoriteRuleManager {
return FavoriteRule.getDefaultRuleIfNull(getByName(ruleName), ruleName);
}
+ private FavoriteRule copyForWrite(FavoriteRule rule) {
+ if (rule.getProject() == null) {
+ rule.setProject(project);
+ }
+ return crud.copyForWrite(rule);
+ }
+
public void resetRule() {
FavoriteRule.getAllDefaultRule().forEach(this::updateRule);
}
@@ -121,7 +128,7 @@ public class FavoriteRuleManager {
}
public void updateRule(List<FavoriteRule.AbstractCondition> conditions, boolean isEnabled, String ruleName) {
- FavoriteRule copy = crud.copyForWrite(getOrDefaultByName(ruleName));
+ FavoriteRule copy = copyForWrite(getOrDefaultByName(ruleName));
copy.setEnabled(isEnabled);
@@ -140,10 +147,11 @@ public class FavoriteRuleManager {
@VisibleForTesting
public void createRule(final FavoriteRule rule) {
- if (getByName(rule.getName()) != null)
+ FavoriteRule copy = copyForWrite(rule);
+ if (getByName(copy.getName()) != null)
return;
- crud.save(rule);
+ crud.save(copy);
}
@VisibleForTesting
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
index 86eb29c09f..f5b105b207 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffset.java
@@ -18,6 +18,8 @@
package org.apache.kylin.metadata.favorite;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -26,6 +28,7 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import org.apache.kylin.metadata.MetadataConstants;
@Getter
@Setter
@@ -37,8 +40,16 @@ public class QueryHistoryIdOffset extends RootPersistentEntity {
@JsonProperty("query_history_stat_id_offset")
private long statMetaUpdateOffset;
+
+ @JsonIgnore
+ private String project;
public QueryHistoryIdOffset(long offset) {
this.offset = offset;
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + getProject() + ResourceStore.QUERY_HISTORY_ID_OFFSET + "/" + uuid + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java
index a96d43afd7..03fa25295b 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/favorite/QueryHistoryIdOffsetManager.java
@@ -24,8 +24,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,12 +40,14 @@ public class QueryHistoryIdOffsetManager {
private final KylinConfig kylinConfig;
private ResourceStore resourceStore;
private String resourceRoot;
+ private String project;
private QueryHistoryIdOffsetManager(KylinConfig kylinConfig, String project) {
if (!UnitOfWork.isAlreadyInTransaction())
logger.info("Initializing QueryHistoryIdOffsetManager with KylinConfig Id: {} for project {}",
System.identityHashCode(kylinConfig), project);
this.kylinConfig = kylinConfig;
+ this.project = project;
resourceStore = ResourceStore.getKylinMetaStore(this.kylinConfig);
this.resourceRoot = "/" + project + ResourceStore.QUERY_HISTORY_ID_OFFSET;
}
@@ -62,9 +65,27 @@ public class QueryHistoryIdOffsetManager {
return this.resourceRoot + "/" + uuid + MetadataConstants.FILE_SURFIX;
}
- public void save(QueryHistoryIdOffset idOffset) {
+ public QueryHistoryIdOffset copyForWrite(QueryHistoryIdOffset idOffset) {
+ if (idOffset.getProject() == null) {
+ idOffset.setProject(project);
+ }
+ return CachedCrudAssist.copyForWrite(idOffset, QUERY_HISTORY_ID_OFFSET_SERIALIZER, null, resourceStore);
+ }
+
+ private void save(QueryHistoryIdOffset idOffset) {
resourceStore.checkAndPutResource(path(idOffset.getUuid()), idOffset, QUERY_HISTORY_ID_OFFSET_SERIALIZER);
}
+
+ public void updateOffset(QueryHistoryIdOffsetUpdater updater) {
+ QueryHistoryIdOffset cached = get();
+ QueryHistoryIdOffset copy = copyForWrite(cached);
+ updater.modify(copy);
+ save(copy);
+ }
+
+ public interface QueryHistoryIdOffsetUpdater {
+ void modify(QueryHistoryIdOffset offset);
+ }
public QueryHistoryIdOffset get() {
List<QueryHistoryIdOffset> queryHistoryIdOffsetList = resourceStore.getAllResources(resourceRoot,
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java
index 1a466ff02c..690c1f7cdf 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/jar/JarInfoManager.java
@@ -76,22 +76,38 @@ public class JarInfoManager {
return String.format(Locale.ROOT, "%s_%s", jarTypeEnum, jarName);
}
+ private JarInfo copyForWrite(JarInfo jarInfo) {
+ return crud.copyForWrite(jarInfo);
+ }
+
public JarInfo createJarInfo(JarInfo jarInfo) {
if (Objects.isNull(jarInfo) || StringUtils.isEmpty(jarInfo.resourceName())) {
throw new IllegalArgumentException("jar info is null or resourceName is null");
}
- if (crud.contains(jarInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_JAR, jarInfo.getJarName());
+ JarInfo copy = copyForWrite(jarInfo);
+ if (crud.contains(copy.resourceName())) {
+ throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_JAR, copy.getJarName());
}
- jarInfo.updateRandomUuid();
- return crud.save(jarInfo);
+ copy.updateRandomUuid();
+ return crud.save(copy);
}
+ /**
+ * @deprecated Use updateJarInfo(JarTypeEnum jarTypeEnum, String jarName, JarInfoUpdater updater) instead.
+ */
+ @Deprecated
public JarInfo updateJarInfo(JarInfo jarInfo) {
- if (!crud.contains(jarInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_JAR, jarInfo.getJarName());
+ return updateJarInfo(jarInfo.getJarType(), jarInfo.getJarName(), jarInfo::copyPropertiesTo);
+ }
+
+ public JarInfo updateJarInfo(JarTypeEnum jarTypeEnum, String jarName, JarInfoUpdater updater) {
+ JarInfo cached = getJarInfo(jarTypeEnum, jarName);
+ if (cached == null) {
+ throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_JAR, jarName);
}
- return crud.save(jarInfo);
+ JarInfo copy = copyForWrite(cached);
+ updater.modify(copy);
+ return crud.save(copy);
}
public JarInfo removeJarInfo(JarTypeEnum jarTypeEnum, String jarName) {
@@ -111,4 +127,8 @@ public class JarInfoManager {
return listJarInfo().stream().filter(jarInfo -> jarTypeEnum == jarInfo.getJarType())
.collect(Collectors.toList());
}
+
+ public interface JarInfoUpdater {
+ void modify(JarInfo copyForWrite);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 005fe4f97a..62ec03ba9f 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -145,6 +145,10 @@ public class ComputedColumnDesc implements Serializable {
return ComputedColumnDesc.CC_PREFIX + columnName;
}
+ public String getIdentityCcName() {
+ return tableIdentity + "." + columnName;
+ }
+
private String handleLegacyCC(String expr, String rootFact, Set<String> aliasSet) {
try {
String ret = CalciteParser.insertAliasInExpr(expr, rootFact);
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java
index 5c22e5754c..21acfc69e5 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModel.java
@@ -27,11 +27,13 @@ import java.util.stream.Collectors;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.MissingRootPersistentEntity;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.Data;
import lombok.EqualsAndHashCode;
@@ -126,4 +128,10 @@ public class FusionModel extends RootPersistentEntity implements Serializable {
}
return null;
}
+
+ @Override
+ public String getResourcePath() {
+ return "/" + project + ResourceStore.FUSION_MODEL_RESOURCE_ROOT + "/" + resourceName()
+ + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java
index 45fdf65aae..6d94db46d2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FusionModelManager.java
@@ -89,14 +89,19 @@ public class FusionModelManager {
return model;
}
+ private FusionModel copyForWrite(FusionModel desc) {
+ return crud.copyForWrite(desc);
+ }
+
public FusionModel createModel(FusionModel desc) {
if (desc == null) {
throw new IllegalArgumentException();
}
- if (crud.contains(desc.resourceName()))
- throw new IllegalArgumentException("Fusion Model '" + desc.getAlias() + "' already exists");
+ FusionModel copy = copyForWrite(desc);
+ if (crud.contains(copy.resourceName()))
+ throw new IllegalArgumentException("Fusion Model '" + copy.getAlias() + "' already exists");
- return crud.save(desc);
+ return crud.save(copy);
}
public String getModelId(NativeQueryRealization realization) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 99c0d6c908..ec47608b85 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -22,6 +22,11 @@ import static org.apache.kylin.common.exception.ServerErrorCode.COLUMN_NOT_EXIST
import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_UPDATE_MODEL;
import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_JOIN_RELATIONSHIP_ERROR;
import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_NOT_EXIST;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_EXPRESSION;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.COMPUTE_COLUMN_NAME;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.INDEX_PLAN;
+import static org.apache.kylin.common.persistence.lock.ModuleLockEnum.ResourceEnum.MODEL_DESC;
+import static org.apache.kylin.common.persistence.lock.ResourcePathParser.JOINER;
import java.io.Serializable;
import java.util.ArrayDeque;
@@ -484,6 +489,11 @@ public class NDataModel extends RootPersistentEntity {
return Collections.unmodifiableMap(aliasMap);
}
+ public void setAliasMap(Map<String, TableRef> aliasMap) {
+ this.aliasMap.clear();
+ this.aliasMap.putAll(aliasMap);
+ }
+
public Set<TableRef> getLookupTables() {
return lookupTableRefs;
}
@@ -1397,6 +1407,20 @@ public class NDataModel extends RootPersistentEntity {
return "/" + project + ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT + "/" + name + MetadataConstants.FILE_SURFIX;
}
+ @Override
+ public List<String> getLockPaths(String ignored) {
+ List<String> lockPaths = new ArrayList<>();
+ lockPaths.add(getResourcePath());
+ lockPaths.add(JOINER.join("", project, INDEX_PLAN.value(), getUuid()));
+ lockPaths.add(JOINER.join("", project, MODEL_DESC.value(), alias));
+ for (ComputedColumnDesc ccDesc : computedColumnDescs) {
+ lockPaths.add(JOINER.join("", project, COMPUTE_COLUMN_NAME.value(), ccDesc.getIdentityCcName()));
+ lockPaths.add(JOINER.join("", project, COMPUTE_COLUMN_EXPRESSION.value(),
+ ccDesc.getInnerExpression().replace('/', '@')));
+ }
+ return lockPaths;
+ }
+
public Collection<NamedColumn> getAllSelectedColumns() {
Set<NamedColumn> selectedColumns = new HashSet<>();
for (NamedColumn namedColumn : allNamedColumns) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
index 8b9f00b0e1..0fe43411d8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModelManager.java
@@ -35,20 +35,21 @@ import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.exception.ModelBrokenException;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -117,6 +118,7 @@ public class NDataModelManager {
model.setJoinTables(entity.getJoinTables());
model.setDependencies(model.calcDependencies());
model.setModelType(entity.getModelType());
+ model.setLastModified(entity.getLastModified());
postModelBrokenEvent(entity);
}
@@ -241,6 +243,7 @@ public class NDataModelManager {
public NDataModel createDataModelDesc(NDataModel model, String owner) {
NDataModel copy = copyForWrite(model);
+ MemoryLockUtils.manuallyLockModule(project, ModuleLockEnum.MODEL, getStore());
checkDuplicateModel(model);
//check model count
@@ -283,21 +286,29 @@ public class NDataModelManager {
}
public NDataModel updateDataModel(String modelId, NDataModelUpdater updater) {
- val cached = getDataModelDesc(modelId);
+ NDataModel cached = getDataModelDesc(modelId);
if (cached == null) {
throw new KylinException(ErrorCodeServer.MODEL_ID_NOT_EXIST, modelId);
}
+ if (cached.isBroken()) {
+ cached = getDataModelDescWithoutInit(modelId);
+ }
val copy = copyForWrite(cached);
updater.modify(copy);
- return updateDataModelDesc(copy);
+ return saveModel(copy);
}
+ /**
+ * @deprecated Use updateDataModel(String modelId, NDataModelUpdater updater) instead.
+ */
+ @Deprecated
public NDataModel updateDataModelDesc(NDataModel desc) {
String name = desc.getUuid();
if (!crud.contains(desc.getUuid())) {
throw new IllegalArgumentException("Model '" + name + "' does not exist.");
}
- return saveModel(desc);
+ desc.init(config, project, getCCRelatedModels(desc));
+ return updateDataModel(desc.getUuid(), desc::copyPropertiesTo);
}
public NDataModel updateDataBrokenModelDesc(NDataModel desc) {
@@ -320,6 +331,10 @@ public class NDataModelManager {
return crud.copyForWrite(nDataModel);
}
+ public NDataModel copy(NDataModel nDataModel) {
+ return copyBySerialization(nDataModel);
+ }
+
/**
* copyBySerialization will not use cache
*/
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
index 64103f9c11..ebc1e893b9 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NTableMetadataManager.java
@@ -31,22 +31,18 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import lombok.val;
-
/**
*/
public class NTableMetadataManager {
@@ -54,8 +50,6 @@ public class NTableMetadataManager {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(NTableMetadataManager.class);
- private static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<>(TableExtDesc.class);
-
public static NTableMetadataManager getInstance(KylinConfig config, String project) {
return config.getManager(project, NTableMetadataManager.class);
}
@@ -153,7 +147,14 @@ public class NTableMetadataManager {
return srcTableCrud.get(tableName);
}
+ public TableDesc copy(TableDesc tableDesc) {
+ return srcTableCrud.copyBySerialization(tableDesc);
+ }
+
public TableDesc copyForWrite(TableDesc tableDesc) {
+ if (tableDesc.getProject() == null) {
+ tableDesc.setProject(project);
+ }
return srcTableCrud.copyForWrite(tableDesc);
}
@@ -161,9 +162,16 @@ public class NTableMetadataManager {
return srcExtCrud.copyForWrite(tableExtDesc);
}
+ /**
+ * @deprecated Use updateTableDesc(String tableName, TableDescUpdater updater) instead.
+ */
+ @Deprecated
public void saveSourceTable(TableDesc srcTable) {
- srcTable.init(project);
- srcTableCrud.save(srcTable);
+ if (srcTableCrud.contains(srcTable.getIdentity())) {
+ updateTableDesc(srcTable.getIdentity(), srcTable::copyPropertiesTo);
+ } else {
+ createTableDesc(srcTable);
+ }
}
public void removeSourceTable(String tableIdentity) {
@@ -182,6 +190,35 @@ public class NTableMetadataManager {
srcTableCrud.reloadAll();
}
+ /**
+ * @deprecated Use updateTableDesc(String tableName, TableDescUpdater updater) instead.
+ */
+ @Deprecated
+ public void updateTableDesc(TableDesc tableDesc) {
+ updateTableDesc(tableDesc.getIdentity(), tableDesc::copyPropertiesTo);
+ }
+
+ public void updateTableDesc(String tableName, TableDescUpdater updater) {
+ TableDesc cached = getTableDesc(tableName);
+ if (cached == null) {
+ throw new IllegalStateException("tableDesc " + tableName + " does not exist");
+ }
+ TableDesc copy = copyForWrite(cached);
+ updater.modify(copy);
+ copy.init(project);
+ srcTableCrud.save(copy);
+ }
+
+ public void createTableDesc(TableDesc srcTable) {
+ srcTable.init(project);
+ TableDesc copy = copyForWrite(srcTable);
+ srcTableCrud.save(copy);
+ }
+
+ public interface TableDescUpdater {
+ void modify(TableDesc copyForWrite);
+ }
+
// ============================================================================
// TableExtDesc methods
// ============================================================================
@@ -239,38 +276,50 @@ public class NTableMetadataManager {
return srcTableCrud.getSerializer();
}
+ /**
+ * @deprecated Use updateTableExt(String tableName, TableExtDescUpdater updater) instead.
+ */
+ @Deprecated
public void saveTableExt(TableExtDesc tableExt) {
- if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
- throw new IllegalArgumentException();
+ if (srcExtCrud.contains(tableExt.getIdentity())) {
+ updateTableExt(tableExt.getIdentity(), tableExt::copyPropertiesTo);
+ } else {
+ createTableExt(tableExt);
}
+ }
- // what is this doing??
- String path = tableExt.getResourcePath();
- ResourceStore store = getStore();
- TableExtDesc t = store.getResource(path, TABLE_EXT_SERIALIZER);
- if (t != null && t.getIdentity() == null)
- store.deleteResource(path);
+ public void mergeAndUpdateTableExt(TableExtDesc origin, TableExtDesc other) {
+ updateTableExt(origin.getIdentity(), copyForWrite -> {
+ copyForWrite.setColumnStats(other.getAllColumnStats());
+ copyForWrite.setSampleRows(other.getSampleRows());
+ copyForWrite.setTotalRows(other.getTotalRows());
+ copyForWrite.setJodID(other.getJodID());
+ if (other.getOriginalSize() != -1) {
+ copyForWrite.setOriginalSize(other.getOriginalSize());
+ }
+ });
+ }
- srcExtCrud.save(tableExt);
+ public void updateTableExt(String tableName, TableExtDescUpdater updater) {
+ TableExtDesc cached = getOrCreateTableExt(tableName);
+ TableExtDesc copy = copyForWrite(cached);
+ updater.modify(copy);
+ srcExtCrud.save(copy);
}
- public void mergeAndUpdateTableExt(TableExtDesc origin, TableExtDesc other) {
- val copyForWrite = srcExtCrud.copyForWrite(origin);
- copyForWrite.setColumnStats(other.getAllColumnStats());
- copyForWrite.setSampleRows(other.getSampleRows());
- copyForWrite.setTotalRows(other.getTotalRows());
- copyForWrite.setJodID(other.getJodID());
- if (other.getOriginalSize() != -1) {
- copyForWrite.setOriginalSize(other.getOriginalSize());
+ public void createTableExt(TableExtDesc tableExt) {
+ if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
+ throw new IllegalArgumentException();
}
- saveTableExt(copyForWrite);
+ TableExtDesc copy = copyForWrite(tableExt);
+ srcExtCrud.save(copy);
}
public void saveOrUpdateTableExt(boolean isUpdate, TableExtDesc tableExt) {
if (isUpdate) {
mergeAndUpdateTableExt(tableExt, tableExt);
} else {
- saveTableExt(tableExt);
+ createTableExt(tableExt);
}
}
@@ -314,11 +363,7 @@ public class NTableMetadataManager {
return result;
}
- public void updateTableDesc(TableDesc tableDesc) {
- if (!srcTableCrud.contains(tableDesc.getIdentity())) {
- throw new IllegalStateException("tableDesc " + tableDesc.getName() + "does not exist");
- }
- saveSourceTable(tableDesc);
+ public interface TableExtDescUpdater {
+ void modify(TableExtDesc copyForWrite);
}
-
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 76dba6c87b..6ff06067d1 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -35,6 +35,9 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -47,9 +50,6 @@ import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import lombok.AllArgsConstructor;
import lombok.Getter;
@@ -374,6 +374,10 @@ public class TableDesc extends RootPersistentEntity implements Serializable, ISo
return project;
}
+ public void setProject(String project) {
+ this.project = project;
+ }
+
public String getName() {
return name == null ? null : name.toUpperCase(Locale.ROOT);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 931b86a5b1..aed99e2747 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -29,9 +29,13 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.base.Strings;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.MetadataConstants;
@@ -40,10 +44,7 @@ import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.base.Strings;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -171,7 +172,7 @@ public class TableExtDesc extends RootPersistentEntity implements Serializable {
}
public void setCardinality(String cardinality) {
- if (null == cardinality)
+ if (StringUtils.isEmpty(cardinality))
return;
String[] cardi = cardinality.split(",");
@@ -191,6 +192,16 @@ public class TableExtDesc extends RootPersistentEntity implements Serializable {
}
}
+ @Override
+ public <T extends RootPersistentEntity> void copyPropertiesTo(T copy) {
+ TableExtDesc extDesc = (TableExtDesc) copy;
+ extDesc.setColumnStats(this.getAllColumnStats()); // copyProperties cannot copy ColumnStats
+ for (Map.Entry<String, String> entry : this.getDataSourceProps().entrySet()) {
+ extDesc.addDataSourceProp(entry.getKey(), entry.getValue());
+ }
+ super.copyPropertiesTo(extDesc);
+ }
+
public enum RowCountStatus {
OK("ok"), TENTATIVE("tentative");
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java
index 821f242aea..5fb0f2f25a 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ImportModelContext.java
@@ -151,7 +151,7 @@ public class ImportModelContext implements AutoCloseable {
List<TableDesc> tables = importTableMetadataManager.listAllTables();
for (TableDesc tableDesc : tables) {
- TableDesc newTable = targetTableMetadataManager.copyForWrite(tableDesc);
+ TableDesc newTable = targetTableMetadataManager.copy(tableDesc);
TableDesc originalTable = targetTableMetadataManager.getTableDesc(newTable.getIdentity());
newTable.setLastModified(System.currentTimeMillis());
if (Objects.isNull(originalTable)) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
index 5b941801fd..7eb5569f09 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
@@ -30,7 +30,11 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
+import org.apache.kylin.common.persistence.lock.ModuleLockEnum;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
@@ -38,9 +42,6 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.val;
public class NProjectManager {
@@ -108,10 +109,11 @@ public class NProjectManager {
ProjectInstance currentProject = getProject(projectName);
if (currentProject == null) {
+ MemoryLockUtils.manuallyLockModule("_global", ModuleLockEnum.PROJECT, getStore());
//circuit breaker
NCircuitBreaker.verifyProjectCreation(listAllProjects().size());
- currentProject = ProjectInstance.create(projectName, owner, description, overrideProps);
+ currentProject = copyForWrite(ProjectInstance.create(projectName, owner, description, overrideProps));
currentProject.initConfig(config);
} else {
throw new IllegalStateException("The project named " + projectName + "already exists");
@@ -166,7 +168,7 @@ public class NProjectManager {
});
}
- public ProjectInstance updateProject(ProjectInstance project) {
+ private ProjectInstance updateProject(ProjectInstance project) {
if (getProject(project.getName()) == null) {
throw new IllegalArgumentException("Project '" + project.getName() + "' does not exist!");
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java
index bd59b47bab..b87512f743 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/resourcegroup/ResourceGroupManager.java
@@ -91,7 +91,7 @@ public class ResourceGroupManager {
public void initResourceGroup() {
if (!resourceGroupInitialized()) {
- save(new ResourceGroup());
+ save(copyForWrite(new ResourceGroup()));
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
index 4cd63edd72..fed4def3c9 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/SourceUsageManager.java
@@ -117,9 +117,14 @@ public class SourceUsageManager {
return crud.copyBySerialization(df);
}
+ public SourceUsageRecord copyForWrite(SourceUsageRecord df) {
+ return crud.copyForWrite(df);
+ }
+
public SourceUsageRecord createSourceUsageRecord(String resourceName, SourceUsageRecord record) {
record.setResPath(concatResourcePath(resourceName));
- return crud.save(record);
+ SourceUsageRecord copy = copyForWrite(record);
+ return crud.save(copy);
}
public SourceUsageRecord updateSourceUsageRecord(String resourceName, SourceUsageRecordUpdater updater) {
@@ -127,7 +132,7 @@ public class SourceUsageManager {
if (record == null) {
record = new SourceUsageRecord();
}
- SourceUsageRecord copy = copy(record);
+ SourceUsageRecord copy = copyForWrite(record);
updater.modify(copy);
return crud.save(copy);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java
index a2913262d8..3eaf5d8199 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/DataParserManager.java
@@ -33,9 +33,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import lombok.extern.slf4j.Slf4j;
@@ -96,18 +95,21 @@ public class DataParserManager {
if (parserInfo == null || StringUtils.isEmpty(parserInfo.getClassName())) {
throw new IllegalArgumentException("data parser info is null or class name is null");
}
- if (crud.contains(parserInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_PARSER, parserInfo.getClassName());
+ DataParserInfo copy = copyForWrite(parserInfo);
+ if (crud.contains(copy.resourceName())) {
+ throw new KylinException(CUSTOM_PARSER_ALREADY_EXISTS_PARSER, copy.getClassName());
}
- parserInfo.updateRandomUuid();
- return crud.save(parserInfo);
+ copy.updateRandomUuid();
+ return crud.save(copy);
}
public DataParserInfo updateDataParserInfo(DataParserInfo parserInfo) {
- if (!crud.contains(parserInfo.resourceName())) {
- throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_PARSER, parserInfo.getClassName());
+ DataParserInfo copy = copyForWrite(parserInfo);
+ if (!crud.contains(copy.resourceName())) {
+ throw new KylinException(CUSTOM_PARSER_NOT_EXISTS_PARSER, copy.getClassName());
}
- return crud.save(parserInfo);
+ parserInfo.copyPropertiesTo(copy);
+ return crud.save(copy);
}
/**
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java
index 06b35a9c67..c2d3a2ee6e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/KafkaConfigManager.java
@@ -75,23 +75,30 @@ public class KafkaConfigManager {
}
return crud.get(id);
}
+
+ private KafkaConfig copyForWrite(KafkaConfig kafkaConfig) {
+ return crud.copyForWrite(kafkaConfig);
+ }
public KafkaConfig createKafkaConfig(KafkaConfig kafkaConfig) {
if (kafkaConfig == null || StringUtils.isEmpty(kafkaConfig.getName())) {
throw new IllegalArgumentException();
}
- if (crud.contains(kafkaConfig.resourceName()))
+ KafkaConfig copy = copyForWrite(kafkaConfig);
+ if (crud.contains(copy.resourceName()))
throw new IllegalArgumentException("Kafka Config '" + kafkaConfig.getName() + "' already exists");
- kafkaConfig.updateRandomUuid();
- return crud.save(kafkaConfig);
+ copy.updateRandomUuid();
+ return crud.save(copy);
}
public KafkaConfig updateKafkaConfig(KafkaConfig kafkaConfig) {
- if (!crud.contains(kafkaConfig.resourceName())) {
+ KafkaConfig copy = copyForWrite(kafkaConfig);
+ if (!crud.contains(copy.resourceName())) {
throw new IllegalArgumentException("Kafka Config '" + kafkaConfig.getName() + "' does not exist.");
}
- return crud.save(kafkaConfig);
+ kafkaConfig.copyPropertiesTo(copy);
+ return crud.save(copy);
}
public KafkaConfig removeKafkaConfig(String tableIdentity) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
index 0bfe31724c..7ccd5ee3f3 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
@@ -83,26 +83,33 @@ public class StreamingManager {
crud.reloadAll();
return listAllStreaming();
}
+
+ private StreamingConfig copyForWrite(StreamingConfig streamingConfig) {
+ return crud.copyForWrite(streamingConfig);
+ }
public StreamingConfig createStreamingConfig(StreamingConfig streamingConfig) throws IOException {
if (streamingConfig == null || StringUtils.isEmpty(streamingConfig.getName())) {
throw new IllegalArgumentException();
}
- if (crud.contains(streamingConfig.resourceName()))
+ StreamingConfig copy = copyForWrite(streamingConfig);
+ if (crud.contains(copy.resourceName()))
throw new IllegalArgumentException("StreamingConfig '" + streamingConfig.getName() + "' already exists");
- streamingConfig.updateRandomUuid();
+ copy.updateRandomUuid();
- return crud.save(streamingConfig);
+ return crud.save(copy);
}
public StreamingConfig updateStreamingConfig(StreamingConfig desc) throws IOException {
if (desc.getUuid() == null || desc.getName() == null) {
throw new IllegalArgumentException("SteamingConfig Illegal.");
}
+ StreamingConfig copy = copyForWrite(desc);
if (!crud.contains(desc.resourceName())) {
throw new IllegalArgumentException("StreamingConfig '" + desc.getName() + "' does not exist.");
}
+ desc.copyPropertiesTo(copy);
return crud.save(desc);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java
index 0a00c78157..215b703144 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/upgrade/GlobalAclVersionManager.java
@@ -22,12 +22,11 @@ import static org.apache.kylin.common.persistence.ResourceStore.UPGRADE;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
public class GlobalAclVersionManager {
private static final Logger logger = LoggerFactory.getLogger(GlobalAclVersionManager.class);
@@ -64,10 +63,15 @@ public class GlobalAclVersionManager {
return crud.exists(GlobalAclVersion.VERSION_KEY_NAME);
}
+ private GlobalAclVersion copyForWrite(GlobalAclVersion globalAclVersion) {
+ return crud.copyForWrite(globalAclVersion);
+ }
+
public void save(GlobalAclVersion globalAclVersion) {
Preconditions.checkNotNull(globalAclVersion.getAclVersion());
+ GlobalAclVersion copy = copyForWrite(globalAclVersion);
if (!exists()) {
- crud.save(globalAclVersion);
+ crud.save(copy);
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java
index a56d4828be..fe459bf6a7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/ManagedUser.java
@@ -51,6 +51,8 @@ import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
+import static org.apache.kylin.common.persistence.ResourceStore.USER_ROOT;
+
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
@Getter
@@ -143,6 +145,11 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails, Us
this.setUuid(RandomUtil.randomUUIDStr());
}
+ @Override
+ public String getResourcePath() {
+ return USER_ROOT + "/" + username;
+ }
+
@Override
public String resourceName() {
return username;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
index bfb324667e..4f40d072df 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/user/NKylinUserManager.java
@@ -31,13 +31,12 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
public class NKylinUserManager {
private static final Logger logger = LoggerFactory.getLogger(NKylinUserManager.class);
@@ -74,6 +73,10 @@ public class NKylinUserManager {
return crud.copyForWrite(user);
}
+ public ManagedUser copy(ManagedUser user) {
+ return crud.copyBySerialization(user);
+ }
+
public KylinConfig getConfig() {
return config;
}
@@ -107,16 +110,30 @@ public class NKylinUserManager {
return users;
}
- public void update(ManagedUser user) {
- ManagedUser exist = crud.get(user.getUsername());
+ public void createUser(ManagedUser user) {
ManagedUser copy = copyForWrite(user);
- if (exist != null) {
- copy.setLastModified(exist.getLastModified());
- copy.setMvcc(exist.getMvcc());
- }
crud.save(copy);
}
+ public void updateUser(String userName, UserUpdater updater) {
+ ManagedUser cached = get(userName);
+ ManagedUser copy = copyForWrite(cached);
+ updater.modify(copy);
+ crud.save(copy);
+ }
+
+ /**
+ * @deprecated Use updateUser(String userName, UserUpdater updater)
+ */
+ @Deprecated
+ public void update(ManagedUser user) {
+ if (exists(user.getUsername())) {
+ updateUser(user.getUsername(), user::copyPropertiesTo);
+ } else {
+ createUser(user);
+ }
+ }
+
public void delete(String username) {
crud.delete(username);
}
@@ -132,4 +149,8 @@ public class NKylinUserManager {
return user.getAuthorities().stream().map(SimpleGrantedAuthority::getAuthority).collect(Collectors.toSet());
}
+
+ public interface UserUpdater {
+ void modify(ManagedUser copy);
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java
index 7a82c08e32..febccc3be6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/NUserGroupManager.java
@@ -137,7 +137,7 @@ public class NUserGroupManager {
String.format(Locale.ROOT, MsgPicker.getMsg().getUserGroupExist(), String.join(",", realGroups)));
}
for (String name : names) {
- UserGroup userGroup = new UserGroup(name);
+ UserGroup userGroup = copyForWrite(new UserGroup(name));
crud.save(userGroup);
}
}
@@ -148,7 +148,7 @@ public class NUserGroupManager {
throw new KylinException(DUPLICATE_USERGROUP_NAME,
String.format(Locale.ROOT, MsgPicker.getMsg().getUserGroupExist(), realGroupName));
}
- UserGroup userGroup = new UserGroup(name);
+ UserGroup userGroup = copyForWrite(new UserGroup(name));
crud.save(userGroup);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
index dc22af1698..aaa428b250 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/usergroup/UserGroup.java
@@ -25,6 +25,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
+import static org.apache.kylin.common.persistence.ResourceStore.USER_GROUP_ROOT;
+
@SuppressWarnings("serial")
@Getter
@Setter
@@ -44,4 +46,9 @@ public class UserGroup extends RootPersistentEntity {
public String resourceName() {
return groupName;
}
+
+ @Override
+ public String getResourcePath() {
+ return USER_GROUP_ROOT + "/" + resourceName();
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java
index f5a29de063..aeda3c2f91 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclManager.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.SpringContext;
@@ -50,8 +51,6 @@ import org.springframework.security.acls.model.PermissionGrantingStrategy;
import org.springframework.security.acls.model.Sid;
import org.springframework.security.core.context.SecurityContextHolder;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.val;
/**
@@ -165,7 +164,7 @@ public class AclManager {
parentAcl = readAclById(record.getParentDomainObjectInfo());
// Caution: Using a copied object before doing init, otherwise the original one will be polluted
- AclRecord recordCopy = crud.copyForWrite(record);
+ AclRecord recordCopy = crud.copyBySerialization(record);
recordCopy.init(parentAcl, aclPermissionFactory, permissionGrantingStrategy);
aclMaps.put(oid, new MutableAclRecord(recordCopy));
@@ -178,7 +177,7 @@ public class AclManager {
if (aclRecord != null) {
throw new AlreadyExistsException(String.format(Locale.ROOT, "ACL of %s exists!", objectIdentity));
}
- AclRecord record = newAclRecord(objectIdentity);
+ AclRecord record = copyForWrite(newAclRecord(objectIdentity));
crud.save(record);
logger.debug("ACL of {} created successfully.", objectIdentity);
@@ -199,11 +198,13 @@ public class AclManager {
logger.debug("ACL of {} deleted successfully.", objectIdentity);
}
+ /**
+ * @deprecated Use updateAcl(MutableAclRecord acl, AclRecordUpdater updater) instead.
+ */
+ @Deprecated
public MutableAcl updateAcl(MutableAcl mutableAcl) {
- AclRecord record = ((MutableAclRecord) mutableAcl).getAclRecord();
- crud.save(record);
- logger.debug("ACL of {} updated successfully.", mutableAcl.getObjectIdentity());
- return mutableAcl;
+ return updateAcl((MutableAclRecord) mutableAcl,
+ ((MutableAclRecord) mutableAcl).getAclRecord()::copyPropertiesTo);
}
// a NULL permission means to delete the ace
@@ -248,6 +249,7 @@ public class AclManager {
val copyForWrite = crud.copyForWrite(acl.getAclRecord());
updater.update(copyForWrite);
crud.save(copyForWrite);
+ logger.debug("ACL of {} updated successfully.", copyForWrite.getObjectIdentity());
return readAcl(acl.getObjectIdentity()); // here we are done
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java
index 21bab12643..1a95f4f57d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/AclRecord.java
@@ -301,6 +301,6 @@ public class AclRecord extends RootPersistentEntity implements Acl, OwnershipAcl
@Override
public String getResourcePath() {
- return ResourceStore.ACL_ROOT + "/" + uuid;
+ return ResourceStore.ACL_ROOT + "/" + resourceName();
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java
index 08813373e1..608eb88fe5 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAcl.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.security;
+import static org.apache.kylin.common.persistence.ResourceStore.ACL_GLOBAL_ROOT;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -121,6 +123,11 @@ public class UserAcl extends RootPersistentEntity {
return CollectionUtils.isNotEmpty(permissionMasks) && permissionMasks.contains(permissionMask);
}
+ @Override
+ public String getResourcePath() {
+ return ACL_GLOBAL_ROOT + "/" + username;
+ }
+
@Override
public String resourceName() {
return username;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
index 7355933597..d19f263141 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/UserAclManager.java
@@ -111,7 +111,7 @@ public class UserAclManager {
public void addPermission(String name, Set<Permission> permissions) {
val username = filterUsername(name);
if (!username.isPresent()) {
- UserAcl userAcl = new UserAcl(name, permissions);
+ UserAcl userAcl = crud.copyForWrite(new UserAcl(name, permissions));
crud.save(userAcl);
} else {
UserAcl userAcl = get(username.get());
@@ -122,7 +122,7 @@ public class UserAclManager {
public void addDataQueryProject(String name, String project) {
val username = filterUsername(name);
if (!username.isPresent()) {
- UserAcl userAcl = new UserAcl(name, Collections.emptySet());
+ UserAcl userAcl = crud.copyForWrite(new UserAcl(name, Collections.emptySet()));
userAcl.addDataQueryProject(project);
crud.save(userAcl);
} else {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java b/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java
index 8dbcb9a2ce..20b3edead2 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/util/BrokenEntityProxy.java
@@ -22,12 +22,11 @@ import java.lang.reflect.Method;
import java.util.Set;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.springframework.cglib.proxy.Enhancer;
import org.springframework.cglib.proxy.MethodInterceptor;
import org.springframework.cglib.proxy.MethodProxy;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.RequiredArgsConstructor;
import lombok.val;
@@ -39,7 +38,7 @@ public class BrokenEntityProxy implements MethodInterceptor {
"toString", "getMvcc", "setMvcc", "setConfig", "getConfig", "getModelAlias", "getModel",
"getRootFactTableName", "setRootFactTableName", "getJoinTables", "setJoinTables", "calcDependencies",
"getDependencies", "setDependencies", "setHandledAfterBroken", "isHandledAfterBroken", "getOwner",
- "setOwner", "isMultiPartitionModel",
+ "setOwner", "isMultiPartitionModel", "setLastModified",
// for broken fusion model
"getModelId", "getModelType", "isStreaming", "isFusionModel", "getModelTypeFromTable", "isAccessible",
"fusionModelStreamingPart", "fusionModelBatchPart", "setModelType", "getLastModified");
@@ -65,6 +64,11 @@ public class BrokenEntityProxy implements MethodInterceptor {
if (method.getName().equals("checkIsNotCachedAndShared")) {
return null;
}
+
+ if (method.getName().equals("getLockPaths")) {
+ return null;
+ }
+
throw new RuntimeException("call on Broken Entity's " + method.getName() + " method");
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
new file mode 100644
index 0000000000..f5ba147c92
--- /dev/null
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.cachesync;
+
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.apache.kylin.metadata.user.ManagedUser;
+import org.apache.kylin.metadata.user.NKylinUserManager;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@MetadataInfo(onlyProps = true)
+class CachedCrudAssistTest {
+
+ @Test
+ void testCopyForWrite() {
+ NKylinUserManager manager = NKylinUserManager.getInstance(getTestConfig());
+
+ ManagedUser newUser = new ManagedUser("a", "a", false);
+
+ ManagedUser brokenUser = new ManagedUser("b", "b", false);
+ brokenUser.setBroken(true);
+ manager.createUser(brokenUser);
+
+ ManagedUser normalUser = new ManagedUser("c", "c", false);
+ manager.createUser(normalUser);
+
+ ManagedUser delUser = new ManagedUser("d", "d", false);
+ delUser.setMvcc(10);
+
+ // null entity
+ Assertions.assertNull(manager.copyForWrite(null));
+
+ // new entity
+ Assertions.assertEquals(newUser, manager.copyForWrite(newUser));
+
+ // broken entity
+ Assertions.assertEquals(brokenUser, manager.copyForWrite(brokenUser));
+
+ // normalEntity
+ Assertions.assertEquals(0, manager.copyForWrite(normalUser).getMvcc());
+
+ // insert a deleted user
+ Assertions.assertThrows(IllegalArgumentException.class, () -> manager.createUser(delUser));
+ }
+}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java
index 0151cac612..0fdefebc67 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/IndexPlanTest.java
@@ -34,9 +34,11 @@ import org.apache.commons.collections.ListUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.guava30.shaded.common.collect.BiMap;
+import org.apache.kylin.guava30.shaded.common.collect.Iterables;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexEntity;
@@ -44,18 +46,14 @@ import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.cube.model.RuleBasedIndex;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import org.apache.kylin.guava30.shaded.common.collect.BiMap;
-import org.apache.kylin.guava30.shaded.common.collect.Iterables;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.val;
import lombok.var;
@@ -314,12 +312,11 @@ public class IndexPlanTest {
// test effect on index plan when project is updated
{
NProjectManager pm = NProjectManager.getInstance(getTestConfig());
- ProjectInstance p = pm.getProject(projectDefault);
- ProjectInstance newP = pm.copyForWrite(p);
- LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(newP.getOverrideKylinProps());
- overrideCopy.put("testkey", "testvalue1");
- newP.setOverrideKylinProps(overrideCopy);
- pm.updateProject(newP);
+ pm.updateProject(projectDefault, copyForWrite -> {
+ LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(copyForWrite.getOverrideKylinProps());
+ overrideCopy.put("testkey", "testvalue1");
+ copyForWrite.setOverrideKylinProps(overrideCopy);
+ });
Assert.assertEquals("testvalue1",
((KylinConfigExt) indexPlan.getConfig()).getExtendedOverrides().get("testkey"));
@@ -327,12 +324,11 @@ public class IndexPlanTest {
{
NProjectManager pm = NProjectManager.getInstance(getTestConfig());
- ProjectInstance p = pm.getProject(projectDefault);
- ProjectInstance newP = pm.copyForWrite(p);
- LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(newP.getOverrideKylinProps());
- overrideCopy.put("testkey", "testvalue2");
- newP.setOverrideKylinProps(overrideCopy);
- pm.updateProject(newP);
+ pm.updateProject(projectDefault, copyForWrite -> {
+ LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(copyForWrite.getOverrideKylinProps());
+ overrideCopy.put("testkey", "testvalue2");
+ copyForWrite.setOverrideKylinProps(overrideCopy);
+ });
Assert.assertEquals("testvalue2",
((KylinConfigExt) indexPlan.getConfig()).getExtendedOverrides().get("testkey"));
@@ -364,12 +360,11 @@ public class IndexPlanTest {
{
//test ProjectInstance trim k-v
NProjectManager pm = NProjectManager.getInstance(getTestConfig());
- ProjectInstance p = pm.getProject(projectDefault);
- ProjectInstance newP = pm.copyForWrite(p);
- LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(newP.getOverrideKylinProps());
- overrideCopy.put(" testkey ", " testvalue2 ");
- newP.setOverrideKylinProps(overrideCopy);
- pm.updateProject(newP);
+ pm.updateProject(projectDefault, copyForWrite -> {
+ LinkedHashMap<String, String> overrideCopy = new LinkedHashMap<>(copyForWrite.getOverrideKylinProps());
+ overrideCopy.put(" testkey ", " testvalue2 ");
+ copyForWrite.setOverrideKylinProps(overrideCopy);
+ });
Assert.assertEquals("testvalue2",
((KylinConfigExt) indexPlan.getConfig()).getExtendedOverrides().get("testkey"));
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java
index dad5d97bfc..eec1754c98 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataLoadingRangeManagerTest.java
@@ -130,10 +130,9 @@ public class NDataLoadingRangeManagerTest extends NLocalFileMetadataTestCase {
String start = "2012-12-25 14:27:14.000";
String end = "2013-01-15 14:27:14.000";
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject("default");
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
- prjManager.updateProject(copy);
+ prjManager.updateProject("default", copyForWrite -> {
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
val loadingRange = createDataLoadingRange(DateFormat.stringToMillis(start), DateFormat.stringToMillis(end));
val ranges = dataLoadingRangeManager.getSegRangesToBuildForNewDataflow(loadingRange);
Assert.assertEquals(4, ranges.size());
@@ -165,13 +164,11 @@ public class NDataLoadingRangeManagerTest extends NLocalFileMetadataTestCase {
String end = "2013-01-15 14:27:14.000";
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject("default");
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().getVolatileRange().setVolatileRangeNumber(3L);
- copy.getSegmentConfig().getVolatileRange().setVolatileRangeEnabled(true);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
-
- prjManager.updateProject(copy);
+ prjManager.updateProject("default", copyForWrite -> {
+ copyForWrite.getSegmentConfig().getVolatileRange().setVolatileRangeNumber(3L);
+ copyForWrite.getSegmentConfig().getVolatileRange().setVolatileRangeEnabled(true);
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
val loadingRange = createDataLoadingRange(DateFormat.stringToMillis(start), DateFormat.stringToMillis(end));
val ranges = dataLoadingRangeManager.getSegRangesToBuildForNewDataflow(loadingRange);
@@ -213,11 +210,10 @@ public class NDataLoadingRangeManagerTest extends NLocalFileMetadataTestCase {
String end = "2012-01-04 20:33:39.000";
val loadingRange = createDataLoadingRange(DateFormat.stringToMillis(start), DateFormat.stringToMillis(end));
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject("default");
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().getAutoMergeTimeRanges().add(AutoMergeTimeEnum.YEAR);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
- prjManager.updateProject(copy);
+ prjManager.updateProject("default", copyForWrite -> {
+ copyForWrite.getSegmentConfig().getAutoMergeTimeRanges().add(AutoMergeTimeEnum.YEAR);
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
val ranges = dataLoadingRangeManager.getSegRangesToBuildForNewDataflow(loadingRange);
Assert.assertEquals(4, ranges.size());
//10/12/24 00:00 - 11/01/01
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java
index f1bf71ddb5..3196105f33 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/model/NDataSegDetailsManagerTest.java
@@ -56,7 +56,7 @@ public class NDataSegDetailsManagerTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(8, details.getLayouts().size());
Assert.assertSame(segment.getConfig().base(), details.getConfig().base());
- details = mgr.upsertForSegment(details);
+ details = mgr.updateDetails(segment, copyForWrite -> {});
details = mgr.getForSegment(segment);
Assert.assertNotNull(details);
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java
index 2cc42c9d91..a5e1b7d230 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/epoch/EnhancedUnitOfWorkTest.java
@@ -26,11 +26,11 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.dbcp.BasicDataSourceFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.StringEntity;
-import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.junit.rule.TransactionExceptedException;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.junit.After;
@@ -42,7 +42,6 @@ import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.springframework.jdbc.core.JdbcTemplate;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -63,8 +62,7 @@ public class EnhancedUnitOfWorkTest extends NLocalFileMetadataTestCase {
+ "@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1,username=sa,password=");
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
- resourceStore.checkAndPutResource("/UUID", new StringEntity(RandomUtil.randomUUIDStr()),
- StringEntity.serializer);
+ resourceStore.createMetaStoreUuidIfNotExist();
return null;
}, "");
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java
index d4e285ac6e..8b9a422f9e 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/NDataModelManagerTest.java
@@ -19,10 +19,12 @@
package org.apache.kylin.metadata.model;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.kylin.common.KapConfig;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
@@ -31,6 +33,7 @@ import org.apache.kylin.common.hystrix.NCircuitBreaker;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.NDataModel.Measure;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -275,4 +278,51 @@ public class NDataModelManagerTest extends NLocalFileMetadataTestCase {
NCircuitBreaker.stop();
}
}
+
+ @Test
+ public void createProjectParallel() {
+ KylinConfig conf = getTestConfig();
+ NDataModelManager manager = NDataModelManager.getInstance(conf, projectDefault);
+ int maxModelNum = manager.listAllModels().size() + 1;
+ getTestConfig().setProperty("kylin.circuit-breaker.threshold.model", String.valueOf(maxModelNum));
+
+ NCircuitBreaker.start(KapConfig.wrap(getTestConfig()));
+
+ NDataModel tmpModel = manager.listAllModels().stream().filter(model -> model.getComputedColumnDescs().isEmpty())
+ .findFirst().get();
+ List<NDataModel> newModels = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ newModels.add(cloneModel(tmpModel));
+ }
+ List<Thread> threads = new ArrayList<>();
+ newModels.forEach(model -> threads.add(new Thread(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ NDataModelManager mManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(),
+ projectDefault);
+ return mManager.createDataModelDesc(model, "ADMIN");
+ }, projectDefault);
+ })));
+ threads.forEach(Thread::start);
+ threads.forEach(thread -> {
+ try {
+ thread.join();
+ } catch (InterruptedException ignored) {
+ }
+ });
+
+ NCircuitBreaker.stop();
+ Assert.assertEquals(maxModelNum, manager.listAllModels().size());
+ }
+
+ private NDataModel cloneModel(NDataModel model) {
+ NDataModel newModel = new NDataModel(model);
+ String uuid = RandomUtil.randomUUIDStr();
+ newModel.setAlias(newModel.getAlias() + uuid);
+ newModel.setProject(model.getProject());
+ newModel.setUuid(uuid);
+ newModel.setMvcc(-1);
+ newModel.setComputedColumnDescs(new ArrayList<>());
+ return newModel;
+ }
+
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
index 4cfb808f39..750fcaf3a2 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata.project;
import java.util.Arrays;
+import java.util.Collections;
import java.util.LinkedHashMap;
import org.apache.hadoop.fs.FileSystem;
@@ -122,4 +123,35 @@ public class NProjectManagerTest extends NLocalFileMetadataTestCase {
project.getConfig().getExtendedOverrides().get("kylin.query.implicit-computed-column-convert"));
}
}
+
+ @Test
+ public void createProjectParallel() throws InterruptedException {
+ KylinConfig conf = getTestConfig();
+ NProjectManager manager = NProjectManager.getInstance(conf);
+ int projectNum = manager.listAllProjects().size();
+ getTestConfig().setProperty("kylin.circuit-breaker.threshold.project", String.valueOf(projectNum + 1));
+
+ NCircuitBreaker.start(KapConfig.wrap(getTestConfig()));
+
+ Thread t1 = new Thread(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ NProjectManager pManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return pManager.createProject("project_tmp_1", "ADMIN", "", new LinkedHashMap<>());
+ }, "_global");
+ });
+ Thread t2 = new Thread(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ NProjectManager pManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return pManager.createProject("project_tmp_2", "ADMIN", "", new LinkedHashMap<>());
+ }, "_global");
+ });
+
+ t1.start();
+ t2.start();
+ t1.join();
+ t2.join();
+
+ NCircuitBreaker.stop();
+ Assert.assertEquals(projectNum + 1, manager.listAllProjects().size());
+ }
}
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java b/src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java
new file mode 100644
index 0000000000..7116330e1a
--- /dev/null
+++ b/src/core-metadata/src/test/java/org/apache/kylin/model/NDataModelTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.model;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.common.persistence.lock.ResourcePathParser;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+class NDataModelTest {
+
+ @Test
+ void testModelLockPath() {
+ NDataModel model = new NDataModel();
+ ComputedColumnDesc cc = new ComputedColumnDesc();
+ cc.setColumnName("cc1");
+ cc.setTableIdentity("ssb.lineOrder");
+ cc.setInnerExpression("a / 1 + a / 2 + a / 4");
+ ReflectionTestUtils.setField(model, "project", "default");
+ ReflectionTestUtils.setField(model, "computedColumnDescs", Collections.singletonList(cc));
+ List<ResourcePathParser.ResourcePath> resourcePaths = new ArrayList<>();
+ for (String lockPath : model.getLockPaths()) {
+ resourcePaths.add(ResourcePathParser.parseResourcePath(lockPath));
+ }
+ Assertions.assertEquals(5, resourcePaths.size());
+ Assertions.assertEquals(cc.getTableIdentity() + "." + cc.getColumnName(), resourcePaths.get(3).getResource1());
+
+ String escapedExpression = "a @ 1 + a @ 2 + a @ 4";
+ Assertions.assertEquals(escapedExpression, resourcePaths.get(4).getResource1());
+ }
+}
diff --git a/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java b/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
index 808af78f71..cacd16a852 100644
--- a/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
+++ b/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
@@ -20,25 +20,21 @@ package org.apache.kylin.sdk.datasource;
import java.util.List;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.sdk.datasource.framework.JdbcConnectorTest;
import org.junit.Assert;
import org.junit.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
public class PushdownRunnerSDKImplTest extends JdbcConnectorTest {
@Test
public void testExecuteQuery() throws Exception {
NProjectManager npr = NProjectManager.getInstance(getTestConfig());
- ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase("SSB");
- npr.updateProject(projectInstance);
+ npr.updateProject("default", copyForWrite -> copyForWrite.setDefaultDatabase("SSB"));
PushDownRunnerSDKImpl pushDownRunnerSDK = new PushDownRunnerSDKImpl();
- pushDownRunnerSDK.init(getTestConfig(), projectInstance.getName());
+ pushDownRunnerSDK.init(getTestConfig(), "default");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
String sql = "select count(*) from LINEORDER";
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
index 4f7f16afb0..1f40e7fc83 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
@@ -370,7 +370,7 @@ public class TableExtService extends BasicService {
}
});
return tableResponse;
- }, project, 1);
+ }, project, 2);
}
private List<Pair<TableDesc, TableExtDesc>> extractTableMeta(String[] tables, String project,
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index b2ae72706b..90cdfa7057 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -61,8 +61,8 @@ import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -74,6 +74,7 @@ import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.AddS3CredentialToSparkBroadcastEventNotifier;
import org.apache.kylin.common.persistence.transaction.TransactionException;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -84,6 +85,19 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.ShellException;
import org.apache.kylin.constants.AclConstants;
+import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.base.Strings;
+import org.apache.kylin.guava30.shaded.common.collect.HashMultimap;
+import org.apache.kylin.guava30.shaded.common.collect.LinkedHashMultimap;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.MapDifference;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Multimap;
+import org.apache.kylin.guava30.shaded.common.collect.SetMultimap;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.graph.Graph;
+import org.apache.kylin.guava30.shaded.common.graph.Graphs;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -170,20 +184,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.base.Strings;
-import org.apache.kylin.guava30.shaded.common.collect.HashMultimap;
-import org.apache.kylin.guava30.shaded.common.collect.LinkedHashMultimap;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.MapDifference;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Multimap;
-import org.apache.kylin.guava30.shaded.common.collect.SetMultimap;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.graph.Graph;
-import org.apache.kylin.guava30.shaded.common.graph.Graphs;
-
import lombok.val;
import lombok.var;
@@ -302,26 +302,19 @@ public class TableService extends BasicService {
if (origTable == null || origTable.getProject() == null) {
nTableDesc.setUuid(RandomUtil.randomUUIDStr());
nTableDesc.setLastModified(0);
+ tableMetaMgr.saveSourceTable(nTableDesc);
} else {
- nTableDesc.setUuid(origTable.getUuid());
- nTableDesc.setLastModified(origTable.getLastModified());
- nTableDesc.setIncrementLoading(origTable.isIncrementLoading());
- nTableDesc.setTableComment(tableDesc.getTableComment());
+ tableMetaMgr.updateTableDesc(nTableDesc.getIdentity(), copyForWrite -> {
+ nTableDesc.copyPropertiesTo(copyForWrite);
+
+ copyForWrite.setUuid(origTable.getUuid());
+ copyForWrite.setLastModified(origTable.getLastModified());
+ copyForWrite.setIncrementLoading(origTable.isIncrementLoading());
+ copyForWrite.setTableComment(tableDesc.getTableComment());
+ });
}
- tableMetaMgr.saveSourceTable(nTableDesc);
if (extDesc != null) {
- TableExtDesc origExt = tableMetaMgr.getTableExtIfExists(tableDesc);
- TableExtDesc nTableExtDesc = new TableExtDesc(extDesc);
- if (origExt == null || origExt.getProject() == null) {
- nTableExtDesc.setUuid(RandomUtil.randomUUIDStr());
- nTableExtDesc.setLastModified(0);
- } else {
- nTableExtDesc.setUuid(origExt.getUuid());
- nTableExtDesc.setLastModified(origExt.getLastModified());
- nTableExtDesc.setMvcc(origExt.getMvcc());
- nTableExtDesc.setOriginalSize(origExt.getOriginalSize());
- }
val colNameMap = Stream.of(nTableDesc.getColumns())
.collect(Collectors.toMap(ColumnDesc::getName, col -> {
try {
@@ -330,11 +323,28 @@ public class TableService extends BasicService {
return Integer.MAX_VALUE;
}
}));
- nTableExtDesc.getAllColumnStats()
- .sort(Comparator.comparing(stat -> colNameMap.getOrDefault(stat.getColumnName(), -1)));
- nTableExtDesc.init(project);
-
- tableMetaMgr.saveTableExt(nTableExtDesc);
+ TableExtDesc origExt = tableMetaMgr.getTableExtIfExists(tableDesc);
+ TableExtDesc nTableExtDesc = new TableExtDesc(extDesc);
+ if (origExt == null || origExt.getProject() == null) {
+ nTableExtDesc.setUuid(RandomUtil.randomUUIDStr());
+ nTableExtDesc.setLastModified(0);
+ nTableExtDesc.getAllColumnStats()
+ .sort(Comparator.comparing(stat -> colNameMap.getOrDefault(stat.getColumnName(), -1)));
+ nTableExtDesc.init(project);
+ tableMetaMgr.saveTableExt(nTableExtDesc);
+ } else {
+ tableMetaMgr.updateTableExt(nTableExtDesc.getIdentity(), copyForWrite -> {
+ nTableExtDesc.copyPropertiesTo(copyForWrite);
+
+ copyForWrite.setUuid(origExt.getUuid());
+ copyForWrite.setLastModified(origExt.getLastModified());
+ copyForWrite.setMvcc(origExt.getMvcc());
+ copyForWrite.setOriginalSize(origExt.getOriginalSize());
+ copyForWrite.getAllColumnStats()
+ .sort(Comparator.comparing(stat -> colNameMap.getOrDefault(stat.getColumnName(), -1)));
+ copyForWrite.init(project);
+ });
+ }
if (!broadcastS3Conf.contains(extDesc.getS3RoleCredentialInfo())) {
addAndBroadcastSparkSession(extDesc.getS3RoleCredentialInfo());
broadcastS3Conf.add(extDesc.getS3RoleCredentialInfo());
@@ -854,8 +864,8 @@ public class TableService extends BasicService {
.collect(Collectors.toSet());
if (tableDesc.getDatabase().equals(projectInstance.getDefaultDatabase())
&& !databases.contains(projectInstance.getDefaultDatabase())) {
- projectInstance.setDefaultDatabase(ProjectInstance.DEFAULT_DATABASE);
- npr.updateProject(projectInstance);
+ npr.updateProject(project,
+ copyForWrite -> copyForWrite.setDefaultDatabase(ProjectInstance.DEFAULT_DATABASE));
}
return tableDesc.getIdentity();
@@ -1229,19 +1239,21 @@ public class TableService extends BasicService {
public Pair<String, List<String>> reloadTable(String projectName, String tableIdentity, boolean needSample,
int maxRows, boolean needBuild, int priority, String yarnQueue) {
aclEvaluate.checkProjectWritePermission(projectName);
- return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- Pair<String, List<String>> pair = new Pair<>();
- List<String> buildingJobs = innerReloadTable(projectName, tableIdentity, needBuild, null);
- pair.setSecond(buildingJobs);
- if (needSample && maxRows > 0) {
- List<String> jobIds = tableSamplingService.sampling(Sets.newHashSet(tableIdentity), projectName,
- maxRows, priority, yarnQueue, null);
- if (CollectionUtils.isNotEmpty(jobIds)) {
- pair.setFirst(jobIds.get(0));
- }
- }
- return pair;
- }, projectName);
+ UnitOfWorkParams<Pair<String, List<String>>> params = UnitOfWorkParams.<Pair<String, List<String>>> builder()
+ .unitName(projectName).processor(() -> {
+ Pair<String, List<String>> pair = new Pair<>();
+ List<String> buildingJobs = innerReloadTable(projectName, tableIdentity, needBuild, null);
+ pair.setSecond(buildingJobs);
+ if (needSample && maxRows > 0) {
+ List<String> jobIds = tableSamplingService.sampling(Sets.newHashSet(tableIdentity), projectName,
+ maxRows, priority, yarnQueue, null);
+ if (CollectionUtils.isNotEmpty(jobIds)) {
+ pair.setFirst(jobIds.get(0));
+ }
+ }
+ return pair;
+ }).build();
+ return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(params);
}
public Pair<String, List<String>> reloadAWSTableCompatibleCrossAccount(String projectName,
diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
index 2d65b47f59..4e6e9c53a6 100644
--- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
+++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
@@ -114,10 +114,6 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase {
ReflectionTestUtils.setField(accessService, "userAclService", userAclService);
ReflectionTestUtils.setField(accessService, "userService", userService);
- val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject(PROJECT);
- val copy = prjManager.copyForWrite(prj);
- prjManager.updateProject(copy);
Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin"));
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString()))
.thenReturn(false);
diff --git a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json
index 65cf8658a2..0dab9315f1 100644
--- a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json
+++ b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b72.json
@@ -6,7 +6,6 @@
"model_type" : "STREAMING",
"alias" : "stream_merge",
"owner" : "ADMIN",
- "model_type": "STREAMING",
"config_last_modifier" : null,
"config_last_modified" : 0,
"description" : "",
diff --git a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json
index 036acaf634..9707ed941d 100644
--- a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json
+++ b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b73.json
@@ -6,7 +6,6 @@
"model_type" : "STREAMING",
"alias" : "stream_merge1",
"owner" : "ADMIN",
- "model_type": "STREAMING",
"config_last_modifier" : null,
"config_last_modified" : 0,
"description" : "",
diff --git a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json
index 877de9a3c5..d87d6e0f0c 100644
--- a/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json
+++ b/src/examples/test_case_data/localmeta/metadata/streaming_test/model_desc/e78a89dd-847f-4574-8afa-8768b4228b74.json
@@ -6,7 +6,6 @@
"model_type" : "STREAMING",
"alias" : "stream_merge2",
"owner" : "ADMIN",
- "model_type": "STREAMING",
"config_last_modifier" : null,
"config_last_modified" : 0,
"description" : "",
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java
index 5ec27dae8f..8f1256860a 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/AutoMergeTest.java
@@ -96,10 +96,9 @@ public class AutoMergeTest extends NLocalFileMetadataTestCase {
val dataflowManager = NDataflowManager.getInstance(getTestConfig(), DEFAULT_PROJECT);
val df = dataflowManager.getDataflowByModelAlias("nmodel_basic");
val prjManager = NProjectManager.getInstance(getTestConfig());
- val prj = prjManager.getProject(DEFAULT_PROJECT);
- val copy = prjManager.copyForWrite(prj);
- copy.getSegmentConfig().setAutoMergeEnabled(true);
- prjManager.updateProject(copy);
+ prjManager.updateProject(DEFAULT_PROJECT, copyForWrite -> {
+ copyForWrite.getSegmentConfig().setAutoMergeEnabled(true);
+ });
SegmentAutoMergeUtil.autoMergeSegments(DEFAULT_PROJECT, df.getUuid(), "ADMIN");
}
diff --git a/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java b/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java
index 1ffa8078ea..09a16ebf76 100644
--- a/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java
+++ b/src/kylin-server-it/src/test/java/org/apache/kylin/event/HAMetadataTest.java
@@ -35,12 +35,14 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ImageDesc;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.StringEntity;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.metadata.JdbcAuditLogStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.tool.MetadataTool;
import org.junit.After;
import org.junit.Assert;
@@ -49,7 +51,6 @@ import org.junit.Ignore;
import org.junit.Test;
import org.springframework.jdbc.core.JdbcTemplate;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -69,6 +70,7 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
+ "@jdbc,driverClassName=org.h2.Driver,url=jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1,username=sa,password=");
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ MemoryLockUtils.lockAndRecord("/UUID", null, false);
resourceStore.checkAndPutResource("/UUID", new StringEntity(RandomUtil.randomUUIDStr()),
StringEntity.serializer);
return null;
@@ -94,6 +96,10 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
queryResourceStore.catchup();
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4", null, false);
resourceStore.checkAndPutResource("/p0/path1", ByteSource.wrap("path1".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path2", ByteSource.wrap("path2".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path3", ByteSource.wrap("path3".getBytes(charset)), -1);
@@ -107,6 +113,10 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
public void testMetadataCatchupWithBackup() throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4", null, false);
resourceStore.checkAndPutResource("/p0/path1", ByteSource.wrap("path1".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path2", ByteSource.wrap("path2".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path3", ByteSource.wrap("path3".getBytes(charset)), -1);
@@ -122,6 +132,13 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path5", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path6", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path7", null, false);
resourceStore.checkAndPutResource("/p0/path1", ByteSource.wrap("path1".getBytes(charset)), 0);
resourceStore.checkAndPutResource("/p0/path2", ByteSource.wrap("path2".getBytes(charset)), 0);
resourceStore.checkAndPutResource("/p0/path3", ByteSource.wrap("path3".getBytes(charset)), 0);
@@ -144,6 +161,10 @@ public class HAMetadataTest extends NLocalFileMetadataTestCase {
public void testMetadata_RemoveAuditLog_Restore() throws Exception {
UnitOfWork.doInTransactionWithRetry(() -> {
val resourceStore = getStore();
+ MemoryLockUtils.lockAndRecord("/p0/path1.json", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path2.json", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path3.json", null, false);
+ MemoryLockUtils.lockAndRecord("/p0/path4.json", null, false);
resourceStore.checkAndPutResource("/_global/project/p0.json", ByteSource
.wrap("{ \"uuid\": \"1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b\"}".getBytes(charset)), -1);
resourceStore.checkAndPutResource("/p0/path1.json",
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 92ddf793a6..29a8076cbf 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -89,9 +89,11 @@ import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
import lombok.val;
+import lombok.extern.slf4j.Slf4j;
@Controller
@RequestMapping(value = "/api/tables", produces = { HTTP_VND_APACHE_KYLIN_JSON })
+@Slf4j
public class NTableController extends NBasicController {
private static final String TABLE = "table";
@@ -231,7 +233,8 @@ public class NTableController extends NBasicController {
LoadTableResponse loadTableResponse = tableExtService.loadTablesWithShortCircuit(tableLoadRequest);
- if (!loadTableResponse.getNeedRealSampling().isEmpty() && Boolean.TRUE.equals(tableLoadRequest.getNeedSampling())) {
+ if (!loadTableResponse.getNeedRealSampling().isEmpty()
+ && Boolean.TRUE.equals(tableLoadRequest.getNeedSampling())) {
TableSamplingService.checkSamplingRows(tableLoadRequest.getSamplingRows());
tableSamplingService.sampling(loadTableResponse.getNeedRealSampling(), tableLoadRequest.getProject(),
tableLoadRequest.getSamplingRows(), tableLoadRequest.getPriority(), tableLoadRequest.getYarnQueue(),
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index e316a9599b..ae7f237265 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -132,6 +132,10 @@ public class ModelRequest extends NDataModel implements ModelInsensitiveRequest
private transient BiFunction<TableDesc, Boolean, Collection<ColumnDesc>> columnsFetcher = TableRef::filterColumns;
+ public BiFunction<TableDesc, Boolean, Collection<ColumnDesc>> getColumnsFetcher() {
+ return columnsFetcher != null ? columnsFetcher : TableRef::filterColumns;
+ }
+
public ModelRequest() {
super();
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index ba699d6c8d..5e2143a39a 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -84,7 +84,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -124,6 +123,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkContext;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
@@ -161,7 +161,6 @@ import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataLayout;
import org.apache.kylin.metadata.cube.model.NDataLoadingRange;
import org.apache.kylin.metadata.cube.model.NDataLoadingRangeManager;
-import org.apache.kylin.metadata.cube.model.NDataSegDetails;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -2024,16 +2023,19 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public NDataModel createModel(String project, ModelRequest modelRequest) {
checkNewModels(project, Lists.newArrayList(modelRequest));
- return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- NDataModel model = saveModel(project, modelRequest);
- modelRequest.setUuid(model.getUuid());
- updateExcludedCheckerResult(project, modelRequest);
- // enable second storage
- if (modelRequest.isWithSecondStorage() && !SecondStorageUtil.isModelEnable(project, model.getId())) {
- SecondStorageUtil.initModelMetaData(project, model.getId());
- }
- return getManager(NDataModelManager.class, project).getDataModelDesc(model.getUuid());
- }, project);
+ UnitOfWorkParams<NDataModel> params = UnitOfWorkParams.<NDataModel> builder().unitName(project)
+ .processor(() -> {
+ NDataModel model = saveModel(project, modelRequest);
+ modelRequest.setUuid(model.getUuid());
+ updateExcludedCheckerResult(project, modelRequest);
+ // enable second storage
+ if (modelRequest.isWithSecondStorage()
+ && !SecondStorageUtil.isModelEnable(project, model.getId())) {
+ SecondStorageUtil.initModelMetaData(project, model.getId());
+ }
+ return getManager(NDataModelManager.class, project).getDataModelDesc(model.getUuid());
+ }).build();
+ return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(params);
}
private NDataModel doCheckBeforeModelSave(String project, ModelRequest modelRequest) {
@@ -2442,10 +2444,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
NDataflow dataflow = dfManger.getDataflow(modelId);
for (String segmentId : segmentIds) {
NDataSegment seg = dataflow.getSegment(segmentId);
- NDataSegDetails segDetails = seg.getSegDetails();
- List<NDataLayout> layouts = new LinkedList<>(segDetails.getAllLayouts());
- layouts.removeIf(layout -> indexIds.contains(layout.getLayoutId()));
- dfManger.updateDataflowDetailsLayouts(seg, layouts);
+ dfManger.updateDataflowDetailsLayouts(seg, indexIds, Collections.emptyList());
}
getManager(NIndexPlanManager.class, project).updateIndexPlan(dataflow.getUuid(),
IndexPlan::removeTobeDeleteIndexIfNecessary);
@@ -3049,6 +3048,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
String[] segmentIds) {
aclEvaluate.checkProjectOperationPermission(project);
SecondStorageJobUtil.validateSegment(project, model, Arrays.asList(segmentIds));
+ UnitOfWork.get().doBeforeUpdate(
+ () -> SecondStorageJobUtil.validateSegment(project, model, Arrays.asList(segmentIds)));
checkSegmentsExistById(model, project, segmentIds);
checkSegmentsStatus(model, project, segmentIds, SegmentStatusEnumToDisplay.LOADING,
SegmentStatusEnumToDisplay.REFRESHING, SegmentStatusEnumToDisplay.MERGING,
@@ -3071,67 +3072,69 @@ public class ModelService extends AbstractModelService implements TableModelSupp
public BuildBaseIndexResponse updateDataModelSemantic(String project, ModelRequest request, boolean isCheckFlat) {
try {
- return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- aclEvaluate.checkProjectWritePermission(project);
- semanticUpdater.expandModelRequest(request);
- checkModelRequest(request);
- checkModelPermission(project, request.getUuid());
- validatePartitionDateColumn(request);
-
- val modelId = request.getUuid();
- val modelManager = getManager(NDataModelManager.class, project);
- val originModel = modelManager.getDataModelDesc(modelId);
-
- val copyModel = modelManager.copyForWrite(originModel);
- UpdateImpact updateImpact = semanticUpdater.updateModelColumns(copyModel, request, true);
- copyModel.init(modelManager.getConfig(), project, modelManager.getCCRelatedModels(copyModel));
-
- BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(originModel,
- needHandleBaseIndexType(request));
-
- preProcessBeforeModelSave(copyModel, project);
- val updated = modelManager.updateDataModelDesc(copyModel);
-
- // 1. delete old internal measure
- // 2. expand new measures
- // the ordering of the two steps is important as tomb internal measure
- // should not be used to construct new expandable measures
- semanticUpdater.deleteExpandableMeasureInternalMeasures(updated);
- semanticUpdater.expandExpandableMeasure(updated);
- preProcessBeforeModelSave(updated, project);
- getManager(NDataModelManager.class, project).updateDataModelDesc(updated);
-
- indexPlanService.updateForMeasureChange(project, modelId, updateImpact.getInvalidMeasures(),
- updateImpact.getReplacedMeasures());
- Set<Integer> affectedSet = updateImpact.getAffectedIds();
- val affectedLayoutSet = getAffectedLayouts(project, modelId, affectedSet);
- if (affectedLayoutSet.size() > 0)
- indexPlanService.reloadLayouts(project, modelId, affectedLayoutSet);
- indexPlanService.clearShardColIfNotDim(project, modelId);
-
- var newModel = modelManager.getDataModelDesc(modelId);
-
- checkIndexColumnExist(project, modelId, originModel);
-
- if (isCheckFlat) {
- checkFlatTableSql(newModel);
- }
-
- val needBuild = semanticUpdater.doHandleSemanticUpdate(project, modelId, originModel,
- request.getStart(), request.getEnd());
-
- updateExcludedCheckerResult(project, request);
- baseIndexUpdater.setSecondStorageEnabled(request.isWithSecondStorage());
- BuildBaseIndexResponse baseIndexResponse = baseIndexUpdater.update(indexPlanService);
- if (!request.isSaveOnly() && (needBuild || baseIndexResponse.hasIndexChange())) {
- semanticUpdater.buildForModel(project, modelId);
- }
- modelChangeSupporters.forEach(listener -> listener.onUpdate(project, modelId));
-
- changeSecondStorageIfNeeded(project, request, () -> !semanticUpdater.isSignificantChange(originModel,
- modelManager.getDataModelDesc(modelId)));
- return baseIndexResponse;
- }, project);
+ UnitOfWorkParams<BuildBaseIndexResponse> params = UnitOfWorkParams.<BuildBaseIndexResponse> builder()
+ .unitName(project).processor(() -> {
+ aclEvaluate.checkProjectWritePermission(project);
+ semanticUpdater.expandModelRequest(request);
+ checkModelRequest(request);
+ checkModelPermission(project, request.getUuid());
+ validatePartitionDateColumn(request);
+
+ val modelId = request.getUuid();
+ val modelManager = getManager(NDataModelManager.class, project);
+ val originModel = modelManager.getDataModelDesc(modelId);
+
+ val copyModel = modelManager.copyForWrite(originModel);
+ UpdateImpact updateImpact = semanticUpdater.updateModelColumns(copyModel, request, true);
+ copyModel.init(modelManager.getConfig(), project, modelManager.getCCRelatedModels(copyModel));
+
+ BaseIndexUpdateHelper baseIndexUpdater = new BaseIndexUpdateHelper(originModel,
+ needHandleBaseIndexType(request));
+
+ preProcessBeforeModelSave(copyModel, project);
+ val updated = modelManager.updateDataModelDesc(copyModel);
+
+ // 1. delete old internal measure
+ // 2. expand new measures
+ // the ordering of the two steps is important as tomb internal measure
+ // should not be used to construct new expandable measures
+ semanticUpdater.deleteExpandableMeasureInternalMeasures(updated);
+ semanticUpdater.expandExpandableMeasure(updated);
+ preProcessBeforeModelSave(updated, project);
+ getManager(NDataModelManager.class, project).updateDataModelDesc(updated);
+
+ indexPlanService.updateForMeasureChange(project, modelId, updateImpact.getInvalidMeasures(),
+ updateImpact.getReplacedMeasures());
+ Set<Integer> affectedSet = updateImpact.getAffectedIds();
+ val affectedLayoutSet = getAffectedLayouts(project, modelId, affectedSet);
+ if (affectedLayoutSet.size() > 0)
+ indexPlanService.reloadLayouts(project, modelId, affectedLayoutSet);
+ indexPlanService.clearShardColIfNotDim(project, modelId);
+
+ var newModel = modelManager.getDataModelDesc(modelId);
+
+ checkIndexColumnExist(project, modelId, originModel);
+
+ if (isCheckFlat) {
+ checkFlatTableSql(newModel);
+ }
+
+ val needBuild = semanticUpdater.doHandleSemanticUpdate(project, modelId, originModel,
+ request.getStart(), request.getEnd());
+
+ updateExcludedCheckerResult(project, request);
+ baseIndexUpdater.setSecondStorageEnabled(request.isWithSecondStorage());
+ BuildBaseIndexResponse baseIndexResponse = baseIndexUpdater.update(indexPlanService);
+ if (!request.isSaveOnly() && (needBuild || baseIndexResponse.hasIndexChange())) {
+ semanticUpdater.buildForModel(project, modelId);
+ }
+ modelChangeSupporters.forEach(listener -> listener.onUpdate(project, modelId));
+
+ changeSecondStorageIfNeeded(project, request, () -> !semanticUpdater
+ .isSignificantChange(originModel, modelManager.getDataModelDesc(modelId)));
+ return baseIndexResponse;
+ }).build();
+ return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(params);
} catch (TransactionException te) {
Throwable root = te.getCause();
if (root instanceof RuntimeException) {
@@ -3363,7 +3366,8 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
NDataModelManager modelMgr = getManager(NDataModelManager.class, project);
semanticUpdater.updateModelColumns(broken, modelRequest);
- NDataModel model = modelMgr.updateDataModelDesc(broken);
+ NDataModel copy = modelManager.copyForWrite(broken);
+ NDataModel model = modelMgr.updateDataModelDesc(copy);
modelMgr.updateDataModel(model.getUuid(), copyForWrite -> {
modelChangeSupporters.forEach(listener -> listener.onUpdateSingle(project, model.getUuid()));
saveDateFormatIfNotExist(project, model.getUuid(), format);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java
index 73b9a94fb7..cd4fae53ef 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/IndexPlanServiceTest.java
@@ -345,12 +345,13 @@ public class IndexPlanServiceTest extends SourceTestCase {
val readySegs = dfManager.getDataflow(id).getSegments(SegmentStatusEnum.READY, SegmentStatusEnum.WARNING);
val last = readySegs.getLatestReadySegment().getSegDetails();
List<NDataLayout> lists = Lists.newArrayList(last.getLayouts());
+ List<Long> toAddLayout = new ArrayList<>();
saved.getRuleBasedIndex().getLayoutIdMapping().forEach(layoutId -> {
if (saved.getLayoutEntity(layoutId).getDimsIds().size() > 1) {
- lists.add(NDataLayout.newDataLayout(last, layoutId));
+ toAddLayout.add(layoutId);
}
});
- dfManager.updateDataflowDetailsLayouts(readySegs.getLatestReadySegment(), lists);
+ dfManager.updateDataflowDetailsLayouts(readySegs.getLatestReadySegment(), Collections.emptyList(), toAddLayout);
indexPlanManager.updateIndexPlan(id, copyForWrite -> {
copyForWrite.markIndexesToBeDeleted(copyForWrite.getId(), saved.getRuleBaseLayouts().stream()
.filter(layoutEntity -> layoutEntity.getColOrder().size() == 3).collect(Collectors.toSet()));
@@ -971,7 +972,9 @@ public class IndexPlanServiceTest extends SourceTestCase {
});
Assert.assertEquals(WARNING, dataflowManager.getDataflow(modelId).getLastSegment().getStatus());
- dataflowManager.updateDataflowDetailsLayouts(df.getLastSegment(), Lists.newArrayList());
+ List<Long> toRemoveLayouts = df.getLastSegment().getSegDetails().getAllLayouts().stream()
+ .map(NDataLayout::getLayoutId).collect(Collectors.toList());
+ dataflowManager.updateDataflowDetailsLayouts(df.getLastSegment(), toRemoveLayouts, Collections.emptyList());
Assert.assertEquals(READY, dataflowManager.getDataflow(modelId).getLastSegment().getStatus());
}
@@ -1270,8 +1273,8 @@ public class IndexPlanServiceTest extends SourceTestCase {
//remove tobedelete layout from seg1
val newDf = dfManager.getDataflow(modelId);
- dfManager.updateDataflowDetailsLayouts(newDf.getSegment(seg1.getId()), layouts1.stream()
- .filter(layout -> layout.getLayoutId() != tobeDeleteLayoutId).collect(Collectors.toList()));
+ dfManager.updateDataflowDetailsLayouts(newDf.getSegment(seg1.getId()),
+ Collections.singletonList(tobeDeleteLayoutId), Collections.emptyList());
//segment1
Assert.assertEquals(1,
indexPlanService.getIndexGraph(getProject(), modelId, 100).getSegmentToComplementCount());
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 58beefba66..8640e901d9 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -1761,8 +1761,7 @@ public class ModelServiceTest extends SourceTestCase {
//remove tobedelete layout from seg1
val newDf = dfManager.getDataflow(modelId);
dfManager.updateDataflowDetailsLayouts(newDf.getSegments().get(0),
- newDf.getSegments().get(0).getLayoutsMap().values().stream()
- .filter(layout -> layout.getLayoutId() != tobeDeleteLayoutId).collect(Collectors.toList()));
+ Collections.singletonList(tobeDeleteLayoutId), Collections.emptyList());
// remove seg2 and tobedelete layout should be cleared from indexplan
val update3 = new NDataflowUpdate(newDf.getUuid());
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
index f18ee8db22..9db76dd0df 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableReloadServiceTest.java
@@ -639,7 +639,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
val reModel = modelManager.getDataModelDescByAlias("nmodel_basic_inner");
Assert.assertNotNull(reModel);
@@ -701,7 +701,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
val reModel = modelManager.getDataModelDescByAlias("nmodel_basic_inner");
Assert.assertNotNull(reModel);
@@ -751,7 +751,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
val modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), PROJECT);
val reModel = modelManager.getDataModelDescByAlias("nmodel_basic_inner");
Assert.assertNotNull(reModel);
@@ -812,7 +812,7 @@ public class TableReloadServiceTest extends CSVSourceTestCase {
UnitOfWork.doInTransactionWithRetry(() -> {
modelService.repairBrokenModel(PROJECT, createModelRequest(copyModel));
return null;
- }, PROJECT, 1);
+ }, PROJECT);
indexPlan = NIndexPlanManager.getInstance(getTestConfig(), PROJECT).getIndexPlan(originModel.getUuid());
Assert.assertEquals(1, indexPlan.getRuleBasedIndex().getAggregationGroups().size());
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index 14b835de6c..baffa9d374 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -612,9 +612,7 @@ public class TableServiceTest extends CSVSourceTestCase {
NProjectManager npr = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
NTableMetadataManager tableManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
"default");
- ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase(removeDB);
- npr.updateProject(projectInstance);
+ npr.updateProject("default", copyForWrite -> copyForWrite.setDefaultDatabase(removeDB));
Assert.assertEquals(removeDB, npr.getDefaultDatabase("default"));
for (TableDesc table : tableManager.listAllTables()) {
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7b6af176aa..60837454a0 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -75,6 +75,7 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.common.util.JsonUtil;
@@ -360,7 +361,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
public void saveQuery(final String creator, final String project, final Query query) throws IOException {
aclEvaluate.checkProjectQueryPermission(project);
Message msg = MsgPicker.getMsg();
- val record = getSavedQueries(creator, project);
+ val record = getSavedQueries(creator, project, true);
+ assert record != null;
List<Query> currentQueries = record.getQueries();
if (currentQueries.stream().map(Query::getName).collect(Collectors.toSet()).contains(query.getName()))
throw new KylinException(SAVE_QUERY_FAILED,
@@ -373,18 +375,31 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
@Transaction(project = 1)
public void removeSavedQuery(final String creator, final String project, final String id) throws IOException {
aclEvaluate.checkProjectQueryPermission(project);
- val record = getSavedQueries(creator, project);
+ val record = getSavedQueries(creator, project, true);
+ assert record != null;
record.setQueries(record.getQueries().stream().filter(q -> !q.getId().equals(id)).collect(Collectors.toList()));
getStore().checkAndPutResource(getQueryKeyById(project, creator), record, QueryRecordSerializer.getInstance());
}
public QueryRecord getSavedQueries(final String creator, final String project) throws IOException {
+ return getSavedQueries(creator, project, false);
+ }
+
+ private QueryRecord getSavedQueries(final String creator, final String project, boolean withWriteLock)
+ throws IOException {
aclEvaluate.checkProjectQueryPermission(project);
if (null == creator) {
return null;
}
- QueryRecord record = getStore().getResource(getQueryKeyById(project, creator),
- QueryRecordSerializer.getInstance());
+ String resPath = getQueryKeyById(project, creator);
+ ResourceStore resourceStore = getStore();
+ QueryRecord record;
+ if (withWriteLock) {
+ record = MemoryLockUtils.doWithLock(null, resPath, false, resourceStore,
+ () -> resourceStore.getResource(resPath, QueryRecordSerializer.getInstance()));
+ } else {
+ record = getStore().getResource(resPath, QueryRecordSerializer.getInstance());
+ }
if (record == null) {
return new QueryRecord();
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 5209745448..200fbb960f 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -94,6 +94,7 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.NativeQueryRealization;
@@ -1240,6 +1241,27 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
}
}
+ @Test
+ public void testSaveAndRemoveQueryWithTransaction() throws IOException {
+ Query query = new Query("test", "default", "test_sql", "test_description");
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ queryService.saveQuery("admin", "default", query);
+ return null;
+ }, "default", 1);
+
+ QueryService.QueryRecord queryRecord = queryService.getSavedQueries("admin", "default");
+ Assert.assertEquals(1, queryRecord.getQueries().size());
+ Assert.assertEquals("test", queryRecord.getQueries().get(0).getName());
+
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ queryService.removeSavedQuery("admin", "default", query.getId());
+ return null;
+ }, "default", 1);
+
+ queryRecord = queryService.getSavedQueries("admin", "default");
+ Assert.assertTrue(queryRecord.getQueries().isEmpty());
+ }
+
@Test
public void testCacheSignature() {
val project = "default";
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java
index a0f384931d..02445a2936 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/util/QueryCacheSignatureUtilTest.java
@@ -32,7 +32,6 @@ import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryMetricsContext;
import org.apache.kylin.rest.response.SQLResponse;
@@ -173,8 +172,9 @@ public class QueryCacheSignatureUtilTest extends NLocalFileMetadataTestCase {
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- TableDesc table = tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
- table.setLastModified(System.currentTimeMillis());
+ tableMetadataManager.updateTableDesc("DEFAULT.TEST_ORDER", copyForWrite -> {
+ copyForWrite.setLastModified(System.currentTimeMillis());
+ });
Assert.assertTrue(QueryCacheSignatureUtil.checkCacheExpired(sqlResponse, project));
}
@@ -194,8 +194,9 @@ public class QueryCacheSignatureUtilTest extends NLocalFileMetadataTestCase {
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- TableDesc table = tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
- table.setLastModified(System.currentTimeMillis());
+ tableMetadataManager.updateTableDesc("DEFAULT.TEST_ORDER", copyForWrite -> {
+ copyForWrite.setLastModified(System.currentTimeMillis());
+ });
Assert.assertTrue(QueryCacheSignatureUtil.checkCacheExpired(sqlResponse, project));
}
@@ -217,8 +218,9 @@ public class QueryCacheSignatureUtilTest extends NLocalFileMetadataTestCase {
NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
project);
- TableDesc table = tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
- table.setLastModified(System.currentTimeMillis());
+ tableMetadataManager.updateTableDesc("DEFAULT.TEST_ORDER", copyForWrite -> {
+ copyForWrite.setLastModified(System.currentTimeMillis());
+ });
Assert.assertTrue(QueryCacheSignatureUtil.checkCacheExpired(sqlResponse, project));
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java
index 2f2dc57d29..33be350e1b 100644
--- a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java
+++ b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklist.java
@@ -24,10 +24,11 @@ import java.io.Serializable;
import java.util.List;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.MetadataConstants;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
import lombok.Data;
@@ -40,6 +41,11 @@ public class SQLBlacklist extends RootPersistentEntity implements Serializable {
public SQLBlacklist() {
updateRandomUuid();
}
+
+ public SQLBlacklist(String project) {
+ this();
+ this.project = project;
+ }
@JsonProperty("project")
private String project;
@@ -135,4 +141,9 @@ public class SQLBlacklist extends RootPersistentEntity implements Serializable {
public String resourceName() {
return project;
}
+
+ @Override
+ public String getResourcePath() {
+ return SQL_BLACKLIST_RESOURCE_ROOT + "/" + resourceName() + MetadataConstants.FILE_SURFIX;
+ }
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java
index e37d492112..e20ab84c1f 100644
--- a/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java
+++ b/src/query/src/main/java/org/apache/kylin/query/blacklist/SQLBlacklistManager.java
@@ -22,14 +22,13 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
public class SQLBlacklistManager {
private static final Logger logger = LoggerFactory.getLogger(SQLBlacklistManager.class);
@@ -102,23 +101,28 @@ public class SQLBlacklistManager {
public SQLBlacklist getSqlBlacklist(String project) {
return crud.get(project);
}
+
+ private SQLBlacklist copyForWrite(SQLBlacklist sqlBlacklist) {
+ return crud.copyForWrite(sqlBlacklist);
+ }
public SQLBlacklist saveSqlBlacklist(SQLBlacklist sqlBlacklist) throws IOException {
SQLBlacklist savedSqlBlacklist = getSqlBlacklist(sqlBlacklist.getProject());
+ SQLBlacklist copy;
if (null != savedSqlBlacklist) {
- savedSqlBlacklist.setBlacklistItems(sqlBlacklist.getBlacklistItems());
- crud.save(savedSqlBlacklist);
+ copy = copyForWrite(savedSqlBlacklist);
+ copy.setBlacklistItems(sqlBlacklist.getBlacklistItems());
} else {
- crud.save(sqlBlacklist);
+ copy = copyForWrite(sqlBlacklist);
}
+ crud.save(copy);
return sqlBlacklist;
}
public SQLBlacklist addSqlBlacklistItem(String project, SQLBlacklistItem sqlBlacklistItem) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
- sqlBlacklist = new SQLBlacklist();
- sqlBlacklist.setProject(project);
+ sqlBlacklist = copyForWrite(new SQLBlacklist(project));
}
sqlBlacklist.addBlacklistItem(sqlBlacklistItem);
crud.save(sqlBlacklist);
@@ -126,7 +130,7 @@ public class SQLBlacklistManager {
}
public SQLBlacklist updateSqlBlacklistItem(String project, SQLBlacklistItem sqlBlacklistItem) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
return null;
}
@@ -142,7 +146,7 @@ public class SQLBlacklistManager {
}
public SQLBlacklist deleteSqlBlacklistItem(String project, String id) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
return null;
}
@@ -152,7 +156,7 @@ public class SQLBlacklistManager {
}
public SQLBlacklist clearBlacklist(String project) throws IOException {
- SQLBlacklist sqlBlacklist = getSqlBlacklist(project);
+ SQLBlacklist sqlBlacklist = copyForWrite(getSqlBlacklist(project));
if (null == sqlBlacklist) {
return null;
}
diff --git a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
index 808f1537b0..0dd1776bcd 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
@@ -108,7 +108,7 @@ public class MetaStoreHealthIndicator implements HealthIndicator, ApplicationLis
@VisibleForTesting
public Health allNodeCheck() {
return UnitOfWork.doInTransactionWithRetry(UnitOfWorkParams.<Health> builder().skipAuditLog(true)
- .unitName(UNIT_NAME).maxRetry(MAX_RETRY).processor(() -> {
+ .unitName(UNIT_NAME).tempLockName(UNIT_NAME).maxRetry(MAX_RETRY).processor(() -> {
ResourceStore store;
try {
store = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
@@ -126,6 +126,7 @@ public class MetaStoreHealthIndicator implements HealthIndicator, ApplicationLis
logger.trace(op);
start = System.currentTimeMillis();
try {
+
store.checkAndPutResource(resourcePath, new StringEntity(uuid), StringEntity.serializer);
checkTime(start, op);
} catch (Exception e) {
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index 5e21a23224..6b3d4d5641 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -142,7 +142,11 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
val scheduler = NDefaultScheduler.getInstance(getProject());
Optional.ofNullable(scheduler.getContext()).ifPresent(context -> {
Optional.ofNullable(context.getRunningJobThread(this)).ifPresent(taskThread -> {
- taskThread.interrupt();
+ // workaround for https://olapio.atlassian.net/browse/KE-41836
+ if (Arrays.stream(taskThread.getStackTrace())
+ .noneMatch(stack -> stack.getClassName().contains("FetcherRunner"))) {
+ taskThread.interrupt();
+ }
context.removeRunningJob(this);
});
});
@@ -522,16 +526,18 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
ResourceStore.dumpKylinProps(tmpDir, props);
} else {
// The way of Updating metadata is CopyOnWrite. So it is safe to use Reference in the value.
- Map<String, RawResource> dumpMap = EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(
- UnitOfWorkParams.<Map> builder().readonly(true).unitName(getProject()).maxRetry(1).processor(() -> {
- Map<String, RawResource> retMap = Maps.newHashMap();
- for (String resPath : getMetadataDumpList(config)) {
- ResourceStore resourceStore = ResourceStore.getKylinMetaStore(config);
- RawResource rawResource = resourceStore.getResource(resPath);
- retMap.put(resPath, rawResource);
- }
- return retMap;
- }).build());
+ Set<String> metadataDumpList = getMetadataDumpList(config);
+ Map<String, RawResource> dumpMap = EnhancedUnitOfWork
+ .doInTransactionWithCheckAndRetry(UnitOfWorkParams.<Map<String, RawResource>> builder()
+ .readonly(true).unitName(getProject()).maxRetry(1).processor(() -> {
+ Map<String, RawResource> retMap = Maps.newHashMap();
+ for (String resPath : metadataDumpList) {
+ ResourceStore resourceStore = ResourceStore.getKylinMetaStore(config);
+ RawResource rawResource = resourceStore.getResource(resPath);
+ retMap.put(resPath, rawResource);
+ }
+ return retMap;
+ }).build());
if (Objects.isNull(dumpMap) || dumpMap.isEmpty()) {
return;
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java
index 314ca23805..fb8684bbd6 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkUpdateMetadataStep.java
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.spark.job;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.val;
import org.apache.hadoop.fs.FileStatus;
@@ -61,10 +62,11 @@ public class NSparkUpdateMetadataStep extends AbstractExecutable {
Preconditions.checkArgument(parent instanceof DefaultExecutableOnModel);
val handler = ((DefaultExecutableOnModel) parent).getHandler();
try {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(UnitOfWorkParams.builder().processor(() -> {
handler.handleFinished();
return null;
- }, context.getEpochId(), handler.getProject());
+ }).retryMoreTimeForDeadLockException(true).epochId(context.getEpochId()).unitName(handler.getProject())
+ .build());
cleanExpiredSnapshot();
return ExecuteResult.createSucceed();
} catch (Throwable throwable) {
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala
index 31eb13f7d7..ffac6bf50f 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotPartitionBuilder.scala
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.spark.builder
-import org.apache.kylin.common.persistence.transaction.UnitOfWork
+import org.apache.kylin.common.persistence.transaction.{UnitOfWork, UnitOfWorkParams}
import org.apache.kylin.common.{KapConfig, KylinConfig}
import org.apache.kylin.engine.spark.utils.LogUtils
import org.apache.kylin.metadata.model.NTableMetadataManager
@@ -62,7 +62,9 @@ class SnapshotPartitionBuilder extends SnapshotBuilder {
copyTable
}
}
- UnitOfWork.doInTransactionWithRetry(new TableUpdateOps, project)
+ val params = UnitOfWorkParams.builder.unitName(project).maxRetry(3).useProjectLock(true)
+ .processor((new TableUpdateOps).asInstanceOf[UnitOfWork.Callback[Nothing]]).build()
+ UnitOfWork.doInTransactionWithRetry(params)
log.info(s"check point partitions for $tableName , partition $partition")
}
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
index 860ed0ad7d..d999411b4c 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
@@ -423,10 +423,11 @@ public class JobManagerTest extends NLocalFileMetadataTestCase {
val segmentRange = new SegmentRange.TimePartitionedSegmentRange(startTime, endTime);
val df = dfm.getDataflow(modelId);
val newSegment = dfm.appendSegment(df, segmentRange, status, partitions);
- newSegment.getMultiPartitions().forEach(partition -> {
+ val copySegment = dfm.copyForWrite(df).getSegment(newSegment.getId());
+ copySegment.getMultiPartitions().forEach(partition -> {
partition.setStatus(PartitionStatusEnum.READY);
});
- return newSegment;
+ return copySegment;
}
private NDataLayout generateLayoutForMultiPartition(String modelId, String segmentId, List<String> partitionValues,
diff --git a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
index d6f4da6cbf..559e3d808e 100644
--- a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
+++ b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
@@ -28,6 +28,8 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.TempMetadataBuilder;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -36,9 +38,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
import lombok.val;
public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
@@ -113,15 +112,16 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
public void testPushdownJdbc() throws Exception {
Connection h2Connection = DriverManager.getConnection("jdbc:h2:mem:db_default", "sa", "");
NProjectManager npr = NProjectManager.getInstance(getTestConfig());
+ npr.updateProject("default", copyForWrite -> {
+ copyForWrite.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ copyForWrite.setOverrideKylinProps(overrideKylinProps);
+ });
ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase("SSB");
- LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
- overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
- projectInstance.setOverrideKylinProps(overrideKylinProps);
- npr.updateProject(projectInstance);
KylinConfigExt config = projectInstance.getConfig();
PushDownRunnerJdbcImpl pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
pushDownRunnerJdbc.init(config, projectInstance.getName());
@@ -135,15 +135,16 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
@Test
public void testProjectPushDownJdbc() throws Exception {
NProjectManager npr = NProjectManager.getInstance(getTestConfig());
+ npr.updateProject("default", copyForWrite -> {
+ copyForWrite.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ copyForWrite.setOverrideKylinProps(overrideKylinProps);
+ });
ProjectInstance projectInstance = npr.getProject("default");
- projectInstance.setDefaultDatabase("SSB");
- LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
- overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
- projectInstance.setOverrideKylinProps(overrideKylinProps);
- npr.updateProject(projectInstance);
KylinConfigExt config = projectInstance.getConfig();
PushDownRunnerJdbcImpl pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
pushDownRunnerJdbc.init(config, projectInstance.getName());
@@ -153,15 +154,16 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
pushDownRunnerJdbc.executeQuery(sql, returnRows, returnColumnMeta, "default");
Assert.assertEquals("1", returnRows.get(0).get(0));
+ npr.updateProject("demo", copyForWrite -> {
+ copyForWrite.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ copyForWrite.setOverrideKylinProps(overrideKylinProps);
+ });
ProjectInstance projectInstance2 = npr.getProject("demo");
- projectInstance2.setDefaultDatabase("SSB");
- overrideKylinProps = Maps.newLinkedHashMap();
- overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
- overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
- projectInstance2.setOverrideKylinProps(overrideKylinProps);
- npr.updateProject(projectInstance2);
config = projectInstance2.getConfig();
pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
pushDownRunnerJdbc.init(config, projectInstance2.getName());
diff --git a/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java b/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
index 777a4830f5..728c816c9e 100644
--- a/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
+++ b/src/streaming/src/main/java/org/apache/kylin/streaming/manager/StreamingJobManager.java
@@ -77,19 +77,24 @@ public class StreamingJobManager {
public void createStreamingJob(NDataModel model, JobTypeEnum jobType) {
StreamingJobMeta job = StreamingJobMeta.create(model, JobStatusEnum.STOPPED, jobType);
- crud.save(job);
+ StreamingJobMeta copy = copyForWrite(job);
+ crud.save(copy);
}
public StreamingJobMeta copy(StreamingJobMeta jobMeta) {
return crud.copyBySerialization(jobMeta);
}
+
+ public StreamingJobMeta copyForWrite(StreamingJobMeta jobMeta) {
+ return crud.copyForWrite(jobMeta);
+ }
public StreamingJobMeta updateStreamingJob(String jobId, NStreamingJobUpdater updater) {
StreamingJobMeta cached = getStreamingJobByUuid(jobId);
if (cached == null) {
return null;
}
- StreamingJobMeta copy = copy(cached);
+ StreamingJobMeta copy = copyForWrite(cached);
updater.modify(copy);
return crud.save(copy);
}
diff --git a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index f3d920d918..d7b1da467e 100644
--- a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
+++ b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -311,8 +311,11 @@ public class MetadataToolHelper {
val projectName = Paths.get(projectPath).getName(0).toString();
val destResources = currentResourceStore.listResourcesRecursively(projectPath);
val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
- UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
- destResources, srcResources, delete), projectName, 1);
+ UnitOfWorkParams<Object> params = UnitOfWorkParams.builder().unitName(projectName).maxRetry(1)
+ .useProjectLock(true).processor(() -> doRestore(currentResourceStore, restoreResourceStore,
+ destResources, srcResources, delete))
+ .build();
+ UnitOfWork.doInTransactionWithRetry(params);
}
} else {
@@ -332,16 +335,20 @@ public class MetadataToolHelper {
Set<String> finalGlobalDestResources = globalDestResources;
- UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
- finalGlobalDestResources, globalSrcResources, delete), UnitOfWork.GLOBAL_UNIT, 1);
+ UnitOfWorkParams<Object> params = UnitOfWorkParams.builder().unitName(UnitOfWork.GLOBAL_UNIT).maxRetry(1)
+ .useProjectLock(true).processor(() -> doRestore(currentResourceStore, restoreResourceStore,
+ finalGlobalDestResources, globalSrcResources, delete))
+ .build();
+ UnitOfWork.doInTransactionWithRetry(params);
val projectPath = FileSystems.getDefault().getSeparator() + project;
val destResources = currentResourceStore.listResourcesRecursively(projectPath);
val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
- UnitOfWork.doInTransactionWithRetry(
- () -> doRestore(currentResourceStore, restoreResourceStore, destResources, srcResources, delete),
- project, 1);
+ params = UnitOfWorkParams.builder().unitName(project).maxRetry(1).useProjectLock(true).processor(
+ () -> doRestore(currentResourceStore, restoreResourceStore, destResources, srcResources, delete))
+ .build();
+ UnitOfWork.doInTransactionWithRetry(params);
}
logger.info("restore successfully");
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java b/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
index 0e6ab4a14c..5e507becea 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
@@ -51,6 +51,7 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.TrashRecord;
+import org.apache.kylin.common.persistence.lock.MemoryLockUtils;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.CliCommandExecutor.CliCmdExecResult;
@@ -200,13 +201,10 @@ public class StorageCleaner implements GarbageCleaner {
}
log.debug("folder {} is collected，detailed -> {}", allFileSystem.getPath(), allFileSystems);
}
- UnitOfWork.doInTransactionWithRetry(() -> {
- collectDeletedProject();
- for (ProjectInstance project : projects) {
- collect(project.getName());
- }
- return null;
- }, UnitOfWork.GLOBAL_UNIT);
+ collectDeletedProject();
+ for (ProjectInstance project : projects) {
+ collect(project.getName());
+ }
long configSurvivalTimeThreshold = timeMachineEnabled ? kylinConfig.getStorageResourceSurvivalTimeThreshold()
: config.getCuboidLayoutSurvivalTimeThreshold();
@@ -307,6 +305,9 @@ public class StorageCleaner implements GarbageCleaner {
ResourceStore threadViewRS = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
RawResource raw = resourceStore.getResource(ResourceStore.METASTORE_TRASH_RECORD);
long mvcc = raw == null ? -1 : raw.getMvcc();
+ // TrashRecord doesn't extend RootPersistentEntity. Let's manually lock it's resource path.
+ MemoryLockUtils.doWithLock(ResourceStore.METASTORE_TRASH_RECORD, false, threadViewRS,
+ () -> null);
threadViewRS.checkAndPutResource(ResourceStore.METASTORE_TRASH_RECORD,
ByteSource.wrap(JsonUtil.writeValueAsBytes(new TrashRecord(trashRecord))), mvcc);
return 0;
@@ -485,13 +486,14 @@ public class StorageCleaner implements GarbageCleaner {
String result = "";
try {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- Set<String> jobTempTables = jobTemps.stream()
- .map(node -> tableCleanerHelper.getJobTransactionalTable(project, node.getName()))
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
+ Set<String> jobTempTables = jobTemps.stream().filter(node -> !node.getName().endsWith(".zip"))
+ .map(node -> tableCleanerHelper.getJobTransactionalTable(project, node.getName(), fs))
.flatMap(Collection::stream).collect(Collectors.toSet());
Set<String> discardTempTables = NExecutableManager.getInstance(config, project)
.getExecutablesByStatus(ExecutableState.DISCARDED).stream()
- .map(e -> tableCleanerHelper.getJobTransactionalTable(project, e.getId()))
+ .map(e -> tableCleanerHelper.getJobTransactionalTable(project, e.getId(), fs))
.flatMap(Collection::stream).collect(Collectors.toSet());
jobTempTables.addAll(discardTempTables);
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java b/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java
index 7c186cfdde..3d5b167096 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/security/KylinPasswordResetCLI.java
@@ -23,6 +23,7 @@ import java.util.Locale;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.util.JsonUtil;
... 208 lines suppressed ...