You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/26 03:08:18 UTC
[kylin] branch kylin5 updated: KYLIN-5378 Add logs when delete auditlog, to improve the diagnosability
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new f7a86ae661 KYLIN-5378 Add logs when delete auditlog, to improve the diagnosability
f7a86ae661 is described below
commit f7a86ae6611e762f7c2a0b819c9b96817a8be104
Author: Jiale He <35...@users.noreply.github.com>
AuthorDate: Fri Oct 28 17:33:24 2022 +0800
KYLIN-5378 Add logs when delete auditlog, to improve the diagnosability
Co-authored-by: Jiale He <ji...@kyligence.io>
---
.../org/apache/kylin/rest/service/MetadataBackupService.java | 8 ++------
.../kylin/common/persistence/metadata/JdbcAuditLogStore.java | 12 ++++++++----
.../java/org/apache/kylin/rest/service/ScheduleService.java | 8 ++++----
.../org/apache/kylin/rest/service/ScheduleServiceTest.java | 7 +++----
4 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java
index 3cb4fe3d28..63c4d22329 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java
@@ -20,10 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.IOException;
import java.time.Clock;
import java.time.LocalDateTime;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -33,20 +30,19 @@ import org.apache.kylin.tool.HDFSMetadataTool;
import org.apache.kylin.tool.MetadataTool;
import org.springframework.stereotype.Service;
+import lombok.SneakyThrows;
import lombok.val;
@Service
public class MetadataBackupService {
- private ExecutorService executors = Executors.newSingleThreadExecutor();
-
@SneakyThrows(IOException.class)
public void backupAll(){
try (SetThreadName ignored = new SetThreadName("MetadataBackupWorker")) {
String[] args = new String[] { "-backup", "-compress", "-dir", getBackupDir() };
backup(args);
- executors.submit(this::rotateAuditLog);
+ rotateAuditLog();
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStore.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStore.java
index 80af263876..20a0eec8e6 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStore.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcAuditLogStore.java
@@ -335,10 +335,14 @@ public class JdbcAuditLogStore implements AuditLogStore {
@Override
public void rotate() {
withTransaction(transactionManager, () -> {
- val maxSize = config.getMetadataAuditLogMaxSize();
- val deletableMaxId = getMaxId() - maxSize + 1;
- log.info("try to delete audit_logs which id less than {}", deletableMaxId);
- jdbcTemplate.update(String.format(Locale.ROOT, DELETE_ID_LESSTHAN_SQL, table), deletableMaxId);
+ val retainMaxSize = config.getMetadataAuditLogMaxSize();
+ val currentMaxId = getMaxId();
+ val deletableMaxId = currentMaxId - retainMaxSize + 1;
+ log.info("try to delete audit_logs which id < {}", deletableMaxId);
+ log.info("retainMaxSize: {}, currentMaxId: {}", retainMaxSize, currentMaxId);
+ val startTime = System.currentTimeMillis();
+ val update = jdbcTemplate.update(String.format(Locale.ROOT, DELETE_ID_LESSTHAN_SQL, table), deletableMaxId);
+ log.info("delete audit_logs count: {}, cost: {}ms", update, System.currentTimeMillis() - startTime);
return null;
});
}
diff --git a/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java b/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java
index eb2f3f2269..05b744c6fe 100644
--- a/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java
+++ b/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java
@@ -66,12 +66,12 @@ public class ScheduleService {
public void routineTask() {
opsCronTimeout = KylinConfig.getInstanceFromEnv().getRoutineOpsTaskTimeOut();
CURRENT_FUTURE.remove();
- long startTime = System.currentTimeMillis();
EpochManager epochManager = EpochManager.getInstance();
try {
+ log.info("Start to work");
+ long startTime = System.currentTimeMillis();
MetricsGroup.hostTagCounterInc(MetricsName.METADATA_OPS_CRON, MetricsCategory.GLOBAL, GLOBAL);
try (SetThreadName ignored = new SetThreadName("RoutineOpsWorker")) {
- log.info("Start to work");
if (epochManager.checkEpochOwner(EpochManager.GLOBAL)) {
executeTask(() -> backupService.backupAll(), "MetadataBackup", startTime);
executeTask(RoutineTool::cleanQueryHistories, "QueryHistoriesCleanup", startTime);
@@ -83,7 +83,7 @@ public class ScheduleService {
executeTask(() -> projectService.garbageCleanup(getRemainingTime(startTime)), "ProjectGarbageCleanup",
startTime);
executeTask(() -> newFastRoutineTool().execute(new String[] { "-c" }), "HdfsCleanup", startTime);
- log.info("Finish to work");
+ log.info("Finish to work, cost {}ms", System.currentTimeMillis() - startTime);
}
} catch (InterruptedException e) {
log.warn("Routine task execution interrupted", e);
@@ -106,7 +106,7 @@ public class ScheduleService {
try {
future.get(remainingTime, TimeUnit.MILLISECONDS);
} catch (ExecutionException e) {
- log.warn("Routine task {} execution failed, reason: {}", taskName, e);
+ log.warn("Routine task {} execution failed, reason:", taskName, e);
}
}
diff --git a/src/job-service/src/test/java/org/apache/kylin/rest/service/ScheduleServiceTest.java b/src/job-service/src/test/java/org/apache/kylin/rest/service/ScheduleServiceTest.java
index f16de263e0..0f735a247d 100644
--- a/src/job-service/src/test/java/org/apache/kylin/rest/service/ScheduleServiceTest.java
+++ b/src/job-service/src/test/java/org/apache/kylin/rest/service/ScheduleServiceTest.java
@@ -76,7 +76,7 @@ public class ScheduleServiceTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testMetadataBackupException() throws Exception {
+ public void testMetadataBackupException() {
getTestConfig().setProperty("kylin.metadata.ops-cron-timeout", "300000ms");
ReflectionTestUtils.setField(scheduleService, "backupService", new MetadataBackupService() {
@SneakyThrows(IOException.class)
@@ -86,12 +86,11 @@ public class ScheduleServiceTest extends NLocalFileMetadataTestCase {
});
EpochManager epochManager = EpochManager.getInstance();
epochManager.updateAllEpochs();
- // thrown.expect(ExecutionException.class);
scheduleService.routineTask();
}
@Test
- public void testRoutineTask() throws Exception {
+ public void testRoutineTask() {
getTestConfig().setProperty("kylin.metadata.ops-cron-timeout", "300000ms");
doNothing().when(projectService).garbageCleanup(anyLong());
EpochManager epochManager = EpochManager.getInstance();
@@ -100,7 +99,7 @@ public class ScheduleServiceTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testTimeoutException() throws Exception {
+ public void testTimeoutException() {
getTestConfig().setProperty("kylin.metadata.ops-cron-timeout", "1000ms");
ReflectionTestUtils.setField(scheduleService, "backupService", new MetadataBackupService() {
@SneakyThrows(Exception.class)