You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/01/24 15:09:13 UTC
(iotdb) branch master updated: Repair unsorted file by compaction (#11887)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7200e217b09 Repair unsorted file by compaction (#11887)
7200e217b09 is described below
commit 7200e217b095d1dd5811b31e2223a68a2e163d78
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Wed Jan 24 23:09:06 2024 +0800
Repair unsorted file by compaction (#11887)
---
.../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 130 +++
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 5 +
.../client/async/handlers/AsyncClientHandler.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 8 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/node/NodeManager.java | 9 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 5 +
.../org/apache/iotdb/db/audit/AuditLogger.java | 1 +
.../iotdb/db/protocol/client/ConfigNodeClient.java | 6 +
.../impl/DataNodeInternalRPCServiceImpl.java | 24 +
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 49 ++
.../config/executor/IConfigTaskExecutor.java | 2 +
.../plan/execution/config/sys/RepairDataTask.java | 42 +
.../db/queryengine/plan/parser/ASTVisitor.java | 12 +
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 5 +
.../plan/statement/sys/RepairDataStatement.java | 62 ++
.../iotdb/db/storageengine/StorageEngine.java | 45 +
.../db/storageengine/dataregion/DataRegion.java | 10 +-
.../impl/ReadPointCompactionPerformer.java | 14 +-
.../RepairUnsortedFileCompactionPerformer.java | 86 ++
.../execute/task/AbstractCompactionTask.java | 9 +-
.../execute/task/CompactionTaskPriorityType.java | 14 +-
.../execute/task/CrossSpaceCompactionTask.java | 5 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +-
.../task/InsertionCrossSpaceCompactionTask.java | 7 +-
.../task/RepairUnsortedFileCompactionTask.java | 239 +++++
.../utils/writer/AbstractCompactionWriter.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 6 +-
.../writer/AbstractInnerCompactionWriter.java | 6 +-
.../utils/writer/FastCrossCompactionWriter.java | 6 +-
.../utils/writer/FastInnerCompactionWriter.java | 6 +-
.../writer/ReadPointCrossCompactionWriter.java | 7 +-
.../writer/ReadPointInnerCompactionWriter.java | 7 +-
.../writer/RepairUnsortedFileCompactionWriter.java | 109 +++
.../dataregion/compaction/repair/RepairLogger.java | 136 +++
.../repair/RepairTaskRecoverLogParser.java | 115 +++
.../compaction/repair/RepairTimePartition.java | 136 +++
.../repair/UnsortedFileRepairTaskScheduler.java | 350 ++++++++
.../compaction/schedule/CompactionScheduler.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 1 -
.../DefaultCompactionTaskComparatorImpl.java | 10 +-
.../RepairUnsortedFileCompactionEstimator.java | 72 ++
.../impl/SizeTieredCompactionSelector.java | 40 +-
.../selector/utils/TsFileResourceCandidate.java | 5 +-
.../compaction/settle/SettleRequestHandler.java | 4 +-
.../dataregion/tsfile/TsFileManager.java | 39 +-
.../TsFileRepairStatus.java} | 14 +-
.../dataregion/tsfile/TsFileResource.java | 15 +
.../tsfile/generator/TsFileNameGenerator.java | 53 +-
.../TsFileOverlapValidationAndRepairTool.java | 6 -
...eCompactionWithFastPerformerValidationTest.java | 4 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../compaction/inner/InnerCompactionTest.java | 1 +
.../compaction/repair/RepairLoggerTest.java | 118 +++
.../repair/RepairUnsortedFileCompactionTest.java | 973 +++++++++++++++++++++
.../utils/MultiTsFileDeviceIteratorTest.java | 12 +-
.../src/main/thrift/confignode.thrift | 3 +
.../src/main/thrift/datanode.thrift | 2 +
64 files changed, 2984 insertions(+), 152 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java
new file mode 100644
index 00000000000..5241531aec0
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertNotNull;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBRepairDataIT {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @Test
+ public void testRepairData() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE root.tesgsg");
+ statement.execute("CREATE TIMESERIES root.testsg.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN");
+ File tsfile = generateUnsortedFile();
+ statement.execute(
+ String.format("load \"%s\" verify=false", tsfile.getParentFile().getAbsolutePath()));
+
+ Assert.assertFalse(validate(statement));
+ statement.execute("REPAIR DATA");
+
+ int waitTimes = 20;
+ for (int i = 0; i < waitTimes; i++) {
+ boolean sorted = validate(statement);
+ if (sorted) {
+ return;
+ }
+ }
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ private File generateUnsortedFile() throws IOException {
+ Path tempDir = Files.createTempDirectory("");
+ tempDir.toFile().deleteOnExit();
+ String tsfileName =
+ TsFileNameGenerator.generateNewTsFileName(System.currentTimeMillis(), 1, 0, 0);
+ File tsfile = new File(tempDir + File.separator + tsfileName);
+ Files.createFile(tsfile.toPath());
+
+ try (TsFileIOWriter writer = new TsFileIOWriter(tsfile)) {
+ writer.startChunkGroup("root.testsg.d1");
+ ChunkWriterImpl chunkWriter =
+ new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT32));
+ chunkWriter.write(2, 1);
+ chunkWriter.write(3, 1);
+ chunkWriter.write(5, 1);
+ chunkWriter.write(4, 1);
+ chunkWriter.sealCurrentPage();
+
+ chunkWriter.writeToFileWriter(writer);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ return tsfile;
+ }
+
+ private boolean validate(Statement statement) throws SQLException {
+ try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM root.testsg.d1")) {
+ assertNotNull(resultSet);
+ long time = Long.MIN_VALUE;
+ while (resultSet.next()) {
+ long currentTime = Long.parseLong(resultSet.getString(TIMESTAMP_STR));
+ if (currentTime <= time) {
+ return false;
+ }
+ time = currentTime;
+ }
+ }
+ return true;
+ }
+}
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
index c828f59ced3..13fb5bc9dfc 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
@@ -182,6 +182,7 @@ keyWords
| RENAME
| RESAMPLE
| RESOURCE
+ | REPAIR
| REPLACE
| REVOKE
| ROLE
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 28cde6c6770..26bff961f3e 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -78,7 +78,7 @@ dclStatement
;
utilityStatement
- : merge | fullMerge | flush | clearCache | settle | explain
+ : merge | fullMerge | flush | clearCache | settle | repairData | explain
| setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource
| showQueries | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding
| loadConfiguration | loadTimeseries | loadFile | removeFile | unloadFile
@@ -938,6 +938,11 @@ settle
: SETTLE (prefixPath|tsFilePath=STRING_LITERAL)
;
+// Repair Data
+repairData
+ : REPAIR DATA (ON (LOCAL | CLUSTER))?
+ ;
+
// Explain
explain
: EXPLAIN selectStatement
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index df4fdd3c08c..5c79f778818 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -989,6 +989,10 @@ MAINTAIN
: M A I N T A I N
;
+REPAIR
+ : R E P A I R
+ ;
+
SCHEMA_REPLICATION_FACTOR
: S C H E M A '_' R E P L I C A T I O N '_' F A C T O R
;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java
index a0f73b456e5..16263484637 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java
@@ -28,6 +28,7 @@ public enum DataNodeRequestType {
FLUSH,
MERGE,
FULL_MERGE,
+ REPAIR_DATA,
LOAD_CONFIGURATION,
SET_SYSTEM_STATUS,
BROADCAST_LATEST_CONFIG_NODE_GROUP,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java
index 139a340079a..3fcd0589961 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java
@@ -294,6 +294,11 @@ public class AsyncDataNodeClientPool {
(AsyncTSStatusRPCHandler)
clientHandler.createAsyncRPCHandler(requestId, targetDataNode));
break;
+ case REPAIR_DATA:
+ client.repairData(
+ (AsyncTSStatusRPCHandler)
+ clientHandler.createAsyncRPCHandler(requestId, targetDataNode));
+ break;
case LOAD_CONFIGURATION:
client.loadConfiguration(
(AsyncTSStatusRPCHandler)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/AsyncClientHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/AsyncClientHandler.java
index 7796252199c..244690c1fbd 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/AsyncClientHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/AsyncClientHandler.java
@@ -233,6 +233,7 @@ public class AsyncClientHandler<Q, R> {
case FULL_MERGE:
case FLUSH:
case CLEAR_CACHE:
+ case REPAIR_DATA:
case LOAD_CONFIGURATION:
case SET_SYSTEM_STATUS:
case UPDATE_REGION_ROUTE_MAP:
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 6d68981ded5..7daef53fccb 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -1411,6 +1411,14 @@ public class ConfigManager implements IManager {
: status;
}
+ @Override
+ public TSStatus repairData() {
+ TSStatus status = confirmLeader();
+ return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ ? RpcUtils.squashResponseStatusList(nodeManager.repairData())
+ : status;
+ }
+
@Override
public TSStatus loadConfiguration() {
TSStatus status = confirmLeader();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index e210a1ff252..bcdc44c677b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -446,6 +446,9 @@ public interface IManager {
/** Clear cache on all DataNodes. */
TSStatus clearCache();
+ /** Check and repair unsorted tsfile by compaction. */
+ TSStatus repairData();
+
/** Load configuration on all DataNodes. */
TSStatus loadConfiguration();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
index 238e6fdb60c..d18f701d1fb 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
@@ -697,6 +697,15 @@ public class NodeManager {
return clientHandler.getResponseList();
}
+ public List<TSStatus> repairData() {
+ Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ AsyncClientHandler<Object, TSStatus> clientHandler =
+ new AsyncClientHandler<>(DataNodeRequestType.REPAIR_DATA, dataNodeLocationMap);
+ AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
public List<TSStatus> loadConfiguration() {
Map<Integer, TDataNodeLocation> dataNodeLocationMap =
configManager.getNodeManager().getRegisteredDataNodeLocations();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 78a47502e7d..b95b949eeea 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -789,6 +789,11 @@ public class ConfigNodeRPCServiceProcessor implements IConfigNodeRPCService.Ifac
return configManager.clearCache();
}
+ @Override
+ public TSStatus repairData() {
+ return configManager.repairData();
+ }
+
@Override
public TSStatus loadConfiguration() {
return configManager.loadConfiguration();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java
index 4f6cecb32a4..b3630db6717 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java
@@ -224,6 +224,7 @@ public class AuditLogger {
case ACTIVATE_TEMPLATE:
case SETTLE:
case INTERNAL_CREATE_TIMESERIES:
+ case REPAIR_DATA:
return AuditLogOperation.DML;
case LIST_USER:
case LIST_ROLE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
index a661171d173..12c6f1d3f90 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
@@ -628,6 +628,12 @@ public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClie
() -> client.clearCache(), status -> !updateConfigNodeLeader(status));
}
+ @Override
+ public TSStatus repairData() throws TException {
+ return executeRemoteCallWithRetry(
+ () -> client.repairData(), status -> !updateConfigNodeLeader(status));
+ }
+
@Override
public TSStatus loadConfiguration() throws TException {
return executeRemoteCallWithRetry(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 23736169581..3969338788b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -57,6 +57,7 @@ import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.consensus.exception.ConsensusGroupAlreadyExistException;
import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException;
import org.apache.iotdb.db.auth.AuthorityChecker;
+import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
@@ -1318,6 +1319,29 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
}
+ @Override
+ public TSStatus repairData() throws TException {
+ if (!storageEngine.isAllSgReady()) {
+ return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, "not all sg is ready");
+ }
+ IoTDBConfig iotdbConfig = IoTDBDescriptor.getInstance().getConfig();
+ if (!iotdbConfig.isEnableSeqSpaceCompaction() || !iotdbConfig.isEnableUnseqSpaceCompaction()) {
+ return RpcUtils.getStatus(
+ TSStatusCode.EXECUTE_STATEMENT_ERROR,
+ "cannot start repair task because inner space compaction is not enabled");
+ }
+ try {
+ if (storageEngine.repairData()) {
+ return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+ } else {
+ return RpcUtils.getStatus(
+ TSStatusCode.EXECUTE_STATEMENT_ERROR, "already have a running repair task");
+ }
+ } catch (StorageEngineException e) {
+ return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage());
+ }
+ }
+
@Override
public TSStatus flush(TFlushReq req) throws TException {
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
index 06ee650f74b..a739c14aa43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
@@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.MergeTask;
+import org.apache.iotdb.db.queryengine.plan.execution.config.sys.RepairDataTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.CreatePipeTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.DropPipeTask;
@@ -137,6 +138,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.SetSpaceQuotaStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.SetThrottleQuotaStatement;
@@ -244,6 +246,12 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon
return new ClearCacheTask(clearCacheStatement);
}
+ @Override
+ public IConfigTask visitRepairData(
+ RepairDataStatement repairDataStatement, MPPQueryContext context) {
+ return new RepairDataTask(repairDataStatement);
+ }
+
@Override
public IConfigTask visitLoadConfiguration(
LoadConfigurationStatement loadConfigurationStatement, MPPQueryContext context) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 36ff96da3a0..df3a9b72ac1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -93,6 +93,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp;
import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
+import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.BatchProcessException;
import org.apache.iotdb.db.exception.StorageEngineException;
@@ -996,6 +997,54 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
return future;
}
+ @Override
+ public SettableFuture<ConfigTaskResult> repairData(boolean onCluster) {
+ SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ TSStatus tsStatus = new TSStatus();
+ if (onCluster) {
+ try (ConfigNodeClient client =
+ CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
+ // Send request to some API server
+ tsStatus = client.repairData();
+ } catch (ClientManagerException | TException e) {
+ future.setException(e);
+ }
+ } else {
+ if (!StorageEngine.getInstance().isAllSgReady()) {
+ future.setException(
+ new IoTDBException(
+ "not all sg is ready", TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()));
+ return future;
+ }
+ IoTDBConfig iotdbConfig = IoTDBDescriptor.getInstance().getConfig();
+ if (!iotdbConfig.isEnableSeqSpaceCompaction()
+ || !iotdbConfig.isEnableUnseqSpaceCompaction()) {
+ future.setException(
+ new IoTDBException(
+ "cannot start repair task because inner space compaction is not enabled",
+ TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()));
+ return future;
+ }
+ try {
+ if (StorageEngine.getInstance().repairData()) {
+ tsStatus = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+ } else {
+ tsStatus =
+ RpcUtils.getStatus(
+ TSStatusCode.EXECUTE_STATEMENT_ERROR, "already have a running repair task");
+ }
+ } catch (Exception e) {
+ tsStatus = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage());
+ }
+ }
+ if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ } else {
+ future.setException(new IoTDBException(tsStatus.message, tsStatus.code));
+ }
+ return future;
+ }
+
@Override
public SettableFuture<ConfigTaskResult> loadConfiguration(boolean onCluster) {
SettableFuture<ConfigTaskResult> future = SettableFuture.create();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
index cc5591c6a95..1209b01bb1c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
@@ -106,6 +106,8 @@ public interface IConfigTaskExecutor {
SettableFuture<ConfigTaskResult> merge(boolean onCluster);
+ SettableFuture<ConfigTaskResult> repairData(boolean onCluster);
+
SettableFuture<ConfigTaskResult> flush(TFlushReq tFlushReq, boolean onCluster);
SettableFuture<ConfigTaskResult> clearCache(boolean onCluster);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/RepairDataTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/RepairDataTask.java
new file mode 100644
index 00000000000..86eed95727b
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/RepairDataTask.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.execution.config.sys;
+
+import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
+import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
+import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataStatement;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class RepairDataTask implements IConfigTask {
+
+ private final RepairDataStatement repairDataStatement;
+
+ public RepairDataTask(RepairDataStatement repairDataStatement) {
+ this.repairDataStatement = repairDataStatement;
+ }
+
+ @Override
+ public ListenableFuture<ConfigTaskResult> execute(IConfigTaskExecutor configTaskExecutor)
+ throws InterruptedException {
+ return configTaskExecutor.repairData(repairDataStatement.isOnCluster());
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 2d31694eab6..9099112b8bf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -185,6 +185,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement;
@@ -3172,6 +3173,17 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return clearCacheStatement;
}
+ // Repair Data
+ @Override
+ public Statement visitRepairData(IoTDBSqlParser.RepairDataContext ctx) {
+ RepairDataStatement repairDataStatement = new RepairDataStatement(StatementType.REPAIR_DATA);
+ if (ctx.CLUSTER() != null && !IoTDBDescriptor.getInstance().getConfig().isClusterMode()) {
+ throw new SemanticException("REPAIR DATA ON CLUSTER is not supported in standalone mode");
+ }
+ repairDataStatement.setOnCluster(ctx.LOCAL() == null);
+ return repairDataStatement;
+ }
+
// Load Configuration
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java
index 2b1cf8a68f8..abccc5c5983 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java
@@ -175,4 +175,5 @@ public enum StatementType {
ALTER_LOGICAL_VIEW,
PIPE_ENRICHED,
+ REPAIR_DATA
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
index c2291e7dcbe..c1b6c1f83e9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
@@ -105,6 +105,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.RepairDataStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement;
@@ -378,6 +379,10 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(clearCacheStatement, context);
}
+ public R visitRepairData(RepairDataStatement repairDataStatement, C context) {
+ return visitStatement(repairDataStatement, context);
+ }
+
public R visitLoadConfiguration(
LoadConfigurationStatement loadConfigurationStatement, C context) {
return visitStatement(loadConfigurationStatement, context);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/RepairDataStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/RepairDataStatement.java
new file mode 100644
index 00000000000..b1a4e46d137
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/RepairDataStatement.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.statement.sys;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
+import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
+import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
+
+import java.util.Collections;
+import java.util.List;
+
+public class RepairDataStatement extends Statement implements IConfigStatement {
+
+ private boolean onCluster;
+
+ public RepairDataStatement(StatementType repairDataType) {
+ this.statementType = repairDataType;
+ }
+
+ public boolean isOnCluster() {
+ return onCluster;
+ }
+
+ public void setOnCluster(boolean onCluster) {
+ this.onCluster = onCluster;
+ }
+
+ @Override
+ public List<PartialPath> getPaths() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public QueryType getQueryType() {
+ return QueryType.WRITE;
+ }
+
+ @Override
+ public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
+ return visitor.visitRepairData(this, context);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 71250a0c54e..ffcd5a0641f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -56,6 +56,8 @@ import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
import org.apache.iotdb.db.storageengine.buffer.ChunkCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairLogger;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.UnsortedFileRepairTaskScheduler;
import org.apache.iotdb.db.storageengine.dataregion.flush.CloseFileListener;
import org.apache.iotdb.db.storageengine.dataregion.flush.FlushListener;
import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy;
@@ -96,6 +98,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
@@ -220,6 +224,7 @@ public class StorageEngine implements IService {
new Thread(
() -> {
checkResults(futures, "StorageEngine failed to recover.");
+ recoverRepairDataScheduleTask();
setAllSgReady(true);
ttlMapForRecover.clear();
},
@@ -557,6 +562,46 @@ public class StorageEngine implements IService {
dataRegionMap.values().forEach(DataRegion::compact);
}
+ /**
+ * check and repair unsorted data by compaction.
+ *
+ * @throws StorageEngineException StorageEngineException
+ */
+ public boolean repairData() throws StorageEngineException {
+ if (CommonDescriptor.getInstance().getConfig().isReadOnly()) {
+ throw new StorageEngineException("Current system mode is read only, does not support merge");
+ }
+ if (!UnsortedFileRepairTaskScheduler.markRepairTaskStart()) {
+ return false;
+ }
+ LOGGER.info("start repair data");
+ List<DataRegion> dataRegionList = new ArrayList<>(dataRegionMap.values());
+ cachedThreadPool.submit(new UnsortedFileRepairTaskScheduler(dataRegionList));
+ return true;
+ }
+
+ /** recover the progress of unfinished repair schedule task */
+ public void recoverRepairDataScheduleTask() {
+ List<DataRegion> dataRegionList = new ArrayList<>(dataRegionMap.values());
+ String repairLogDirPath =
+ IoTDBDescriptor.getInstance().getConfig().getSystemDir()
+ + File.separator
+ + RepairLogger.repairLogDir;
+ File repairLogDir = new File(repairLogDirPath);
+ if (!repairLogDir.exists() || !repairLogDir.isDirectory()) {
+ return;
+ }
+ File[] files = repairLogDir.listFiles();
+ List<File> fileList =
+ Stream.of(files == null ? new File[0] : files)
+ .filter(f -> f.getName().endsWith(RepairLogger.repairLogSuffix) && f.isFile())
+ .collect(Collectors.toList());
+ if (!fileList.isEmpty()) {
+ UnsortedFileRepairTaskScheduler.markRepairTaskStart();
+ cachedThreadPool.submit(new UnsortedFileRepairTaskScheduler(dataRegionList, fileList.get(0)));
+ }
+ }
+
public void operateFlush(TFlushReq req) {
if (req.storageGroups == null) {
StorageEngine.getInstance().syncCloseAllProcessor();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 8ad98d45eae..2a4575bf3e7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2444,7 +2444,7 @@ public class DataRegion implements IDataRegionForQuery {
// the name of this variable is trySubmitCount, because the task submitted to the queue could
// be
// evicted due to the low priority of the task
- CompactionScheduler.lockCompactionSelection();
+ CompactionScheduler.sharedLockCompactionSelection();
try {
for (long timePartition : timePartitions) {
trySubmitCount +=
@@ -2453,7 +2453,7 @@ public class DataRegion implements IDataRegionForQuery {
} catch (Throwable e) {
logger.error("Meet error in compaction schedule.", e);
} finally {
- CompactionScheduler.unlockCompactionSelection();
+ CompactionScheduler.sharedUnlockCompactionSelection();
}
}
if (summary.hasSubmitTask()) {
@@ -2464,7 +2464,7 @@ public class DataRegion implements IDataRegionForQuery {
protected int executeInsertionCompaction(List<Long> timePartitions) {
int trySubmitCount = 0;
- CompactionScheduler.lockCompactionSelection();
+ CompactionScheduler.sharedLockCompactionSelection();
try {
while (true) {
int currentSubmitCount = 0;
@@ -2484,7 +2484,7 @@ public class DataRegion implements IDataRegionForQuery {
} catch (Throwable e) {
logger.error("Meet error in compaction schedule.", e);
} finally {
- CompactionScheduler.unlockCompactionSelection();
+ CompactionScheduler.sharedUnlockCompactionSelection();
}
return trySubmitCount;
}
@@ -2534,9 +2534,11 @@ public class DataRegion implements IDataRegionForQuery {
/** merge file under this database processor */
public int compact() {
writeLock("merge");
+ CompactionScheduler.exclusiveLockCompactionSelection();
try {
return executeCompaction();
} finally {
+ CompactionScheduler.exclusiveUnlockCompactionSelection();
writeUnlock();
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java
index 3c40ebe8da1..e0b5f4bd8ec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java
@@ -67,15 +67,15 @@ public class ReadPointCompactionPerformer
@SuppressWarnings("squid:S1068")
private final Logger logger = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME);
- private List<TsFileResource> seqFiles = Collections.emptyList();
- private List<TsFileResource> unseqFiles = Collections.emptyList();
+ protected List<TsFileResource> seqFiles = Collections.emptyList();
+ protected List<TsFileResource> unseqFiles = Collections.emptyList();
private static final int SUB_TASK_NUM =
IoTDBDescriptor.getInstance().getConfig().getSubCompactionTaskNum();
private CompactionTaskSummary summary;
- private List<TsFileResource> targetFiles = Collections.emptyList();
+ protected List<TsFileResource> targetFiles = Collections.emptyList();
public ReadPointCompactionPerformer(
List<TsFileResource> seqFiles,
@@ -269,11 +269,7 @@ public class ReadPointCompactionPerformer
while (reader.hasNextBatch()) {
TsBlock tsBlock = reader.nextBatch();
if (isAligned) {
- writer.write(
- tsBlock.getTimeColumn(),
- tsBlock.getValueColumns(),
- subTaskId,
- tsBlock.getPositionCount());
+ writer.write(tsBlock, subTaskId);
} else {
IPointReader pointReader = tsBlock.getTsBlockSingleColumnIterator();
while (pointReader.hasNextTimeValuePair()) {
@@ -283,7 +279,7 @@ public class ReadPointCompactionPerformer
}
}
- private AbstractCompactionWriter getCompactionWriter(
+ protected AbstractCompactionWriter getCompactionWriter(
List<TsFileResource> seqFileResources,
List<TsFileResource> unseqFileResources,
List<TsFileResource> targetFileResources)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
new file mode 100644
index 00000000000..163987ce434
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl;
+
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.RepairUnsortedFileCompactionWriter;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+/** Used for fixing files which contains internal unsorted data */
+public class RepairUnsortedFileCompactionPerformer extends ReadPointCompactionPerformer {
+
+ private final boolean rewriteFile;
+
+ public RepairUnsortedFileCompactionPerformer(boolean rewriteFile) {
+ super();
+ this.rewriteFile = rewriteFile;
+ }
+
+ @Override
+ protected AbstractCompactionWriter getCompactionWriter(
+ List<TsFileResource> seqFileResources,
+ List<TsFileResource> unseqFileResources,
+ List<TsFileResource> targetFileResources)
+ throws IOException {
+ return new RepairUnsortedFileCompactionWriter(targetFileResources.get(0));
+ }
+
+ @Override
+ public void perform() throws Exception {
+ if (rewriteFile) {
+ super.perform();
+ } else {
+ prepareTargetFile();
+ }
+ }
+
+ private void prepareTargetFile() throws IOException {
+ TsFileResource seqSourceFile = seqFiles.get(0);
+ TsFileResource targetFile = targetFiles.get(0);
+ Files.createLink(targetFile.getTsFile().toPath(), seqSourceFile.getTsFile().toPath());
+ ITimeIndex timeIndex = seqSourceFile.getTimeIndex();
+ if (timeIndex instanceof DeviceTimeIndex) {
+ targetFile.setTimeIndex(timeIndex);
+ } else {
+ targetFile.setTimeIndex(seqSourceFile.buildDeviceTimeIndex());
+ }
+ if (seqSourceFile.modFileExists()) {
+ Files.createLink(
+ new File(seqSourceFile.getCompactionModFile().getFilePath()).toPath(),
+ new File(seqSourceFile.getModFile().getFilePath()).toPath());
+ }
+ }
+
+ @Override
+ public void setSourceFiles(List<TsFileResource> sourceFiles) {
+ if (sourceFiles.get(0).isSeq()) {
+ seqFiles = sourceFiles;
+ } else {
+ unseqFiles = sourceFiles;
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index a524a4e7b57..77bb672ef98 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.utils.validate.TsFileValidator;
@@ -134,7 +135,7 @@ public abstract class AbstractCompactionTask {
public void handleTaskCleanup() {}
- protected void printLogWhenException(Logger logger, Exception e) {
+ protected void handleException(Logger logger, Exception e) {
if (e instanceof CompactionLastTimeCheckFailedException
|| e instanceof CompactionValidationFailedException) {
logger.error(
@@ -143,6 +144,10 @@ public abstract class AbstractCompactionTask {
storageGroupName,
dataRegionId,
e.getMessage());
+ // these exceptions generally caused by unsorted data, mark all source files as NEED_TO_REPAIR
+ for (TsFileResource resource : getAllSourceTsFiles()) {
+ resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR);
+ }
} else if (e instanceof InterruptedException) {
logger.warn("{}-{} [Compaction] Compaction interrupted", storageGroupName, dataRegionId);
Thread.currentThread().interrupt();
@@ -394,7 +399,7 @@ public abstract class AbstractCompactionTask {
CompactionTaskType taskType = getCompactionTaskType();
boolean needToValidateTsFileCorrectness = taskType != CompactionTaskType.INSERTION;
boolean needToValidatePartitionSeqSpaceOverlap =
- getCompactionTaskType() != CompactionTaskType.INNER_UNSEQ;
+ !targetFiles.isEmpty() && targetFiles.get(0).isSeq();
TsFileValidator validator = TsFileValidator.getInstance();
if (needToValidatePartitionSeqSpaceOverlap) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java
index 79aea997f58..a2b7b5f67ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java
@@ -21,12 +21,22 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
public enum CompactionTaskPriorityType {
/** default compaction task type */
- NORMAL,
+ NORMAL(10),
/**
* in either of the following situations: 1. the TsFile has .mods file whose size exceeds 50 MB.
* 2. the TsFile has .mods file and the disk availability rate is lower than the
* disk_space_warning_threshold.
*/
- MOD_SETTLE
+ MOD_SETTLE(20),
+ REPAIR_DATA(30);
+ private final int value;
+
+ CompactionTaskPriorityType(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
index f5a4071503a..f2e7d4dd88d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -204,8 +204,7 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
selectedSequenceFiles,
selectedUnsequenceFiles,
targetTsfileResourceList,
- timePartition,
- true);
+ timePartition);
// find empty target files and add log
for (TsFileResource targetResource : targetTsfileResourceList) {
@@ -278,7 +277,7 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
}
} catch (Exception e) {
isSuccess = false;
- printLogWhenException(LOGGER, e);
+ handleException(LOGGER, e);
recover();
} finally {
releaseAllLocks();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 0f46242093c..5a5538bd6bc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -62,7 +62,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
protected int sumOfCompactionCount;
protected long maxFileVersion;
protected int maxCompactionCount;
- private File logFile;
+ protected File logFile;
protected List<TsFileResource> targetTsFileList;
protected boolean[] isHoldingWriteLock;
protected long maxModsFileSize;
@@ -150,7 +150,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
createSummary();
}
- private void prepare() throws IOException {
+ protected void prepare() throws IOException {
targetTsFileResource =
TsFileNameGenerator.getInnerCompactionTargetFileResource(
selectedTsFileResourceList, sequence);
@@ -203,17 +203,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
performer.setSummary(summary);
performer.perform();
- CompactionUtils.updateProgressIndex(
- targetTsFileList, selectedTsFileResourceList, Collections.emptyList());
- CompactionUtils.moveTargetFile(
- targetTsFileList, true, storageGroupName + "-" + dataRegionId);
-
- LOGGER.info(
- "{}-{} [InnerSpaceCompactionTask] start to rename mods file",
- storageGroupName,
- dataRegionId);
- CompactionUtils.combineModsInInnerCompaction(
- selectedTsFileResourceList, targetTsFileResource);
+ prepareTargetFiles();
if (Thread.currentThread().isInterrupted() || summary.isCancel()) {
throw new InterruptedException(
@@ -230,8 +220,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
sequence ? selectedTsFileResourceList : Collections.emptyList(),
sequence ? Collections.emptyList() : selectedTsFileResourceList,
targetTsFileList,
- timePartition,
- sequence);
+ timePartition);
if (targetTsFileResource.isDeleted()) {
compactionLogger.logEmptyTargetFile(targetTsFileResource);
@@ -275,7 +264,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
targetTsFileResource.getDatabaseName(),
targetTsFileResource.getDataRegionId(),
targetTsFileResource.getTsFile().length(),
- sequence,
+ targetTsFileResource.isSeq(),
targetTsFileResource.getTsFile().getName());
} else {
// target resource is empty after compaction, then delete it
@@ -303,7 +292,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
}
} catch (Exception e) {
isSuccess = false;
- printLogWhenException(LOGGER, e);
+ handleException(LOGGER, e);
recover();
} finally {
releaseAllLocks();
@@ -311,6 +300,18 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
return isSuccess;
}
+ protected void prepareTargetFiles() throws IOException {
+ CompactionUtils.updateProgressIndex(
+ targetTsFileList, selectedTsFileResourceList, Collections.emptyList());
+ CompactionUtils.moveTargetFile(targetTsFileList, true, storageGroupName + "-" + dataRegionId);
+
+ LOGGER.info(
+ "{}-{} [InnerSpaceCompactionTask] start to rename mods file",
+ storageGroupName,
+ dataRegionId);
+ CompactionUtils.combineModsInInnerCompaction(selectedTsFileResourceList, targetTsFileResource);
+ }
+
public void recover() {
try {
if (needRecoverTaskInfoFromLogFile) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index f4662951705..e4c5ac2deac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -179,14 +179,14 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
String.format("%.2f", costTime));
} catch (Exception e) {
isSuccess = false;
- printLogWhenException(LOGGER, e);
+ handleException(LOGGER, e);
recover();
} finally {
releaseAllLocks();
try {
Files.deleteIfExists(logFile.toPath());
} catch (IOException e) {
- printLogWhenException(LOGGER, e);
+ handleException(LOGGER, e);
}
targetFile.setStatus(TsFileResourceStatus.NORMAL);
}
@@ -226,6 +226,7 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
}
targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
targetFile.deserialize();
+ targetFile.setProgressIndex(unseqFileToInsert.getMaxProgressIndexAfterClose());
}
private boolean recoverTaskInfoFromLogFile() throws IOException {
@@ -271,7 +272,7 @@ public class InsertionCrossSpaceCompactionTask extends AbstractCompactionTask {
try {
Files.deleteIfExists(logFile.toPath());
} catch (IOException e) {
- printLogWhenException(LOGGER, e);
+ handleException(LOGGER, e);
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
new file mode 100644
index 00000000000..f9b7b782a0f
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
+
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.RepairUnsortedFileCompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.RepairUnsortedFileCompactionEstimator;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * Repair the internal unsorted file by compaction and move it to unSequence space after compaction
+ * whether the source TsFileResource is sequence or not.
+ */
+public class RepairUnsortedFileCompactionTask extends InnerSpaceCompactionTask {
+
+ private final TsFileResource sourceFile;
+ private final boolean rewriteFile;
+ private CountDownLatch latch;
+
+ public RepairUnsortedFileCompactionTask(
+ long timePartition,
+ TsFileManager tsFileManager,
+ TsFileResource sourceFile,
+ boolean sequence,
+ long serialId) {
+ super(
+ timePartition,
+ tsFileManager,
+ Collections.singletonList(sourceFile),
+ sequence,
+ new RepairUnsortedFileCompactionPerformer(true),
+ serialId,
+ CompactionTaskPriorityType.REPAIR_DATA);
+ this.sourceFile = sourceFile;
+ this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator();
+ this.rewriteFile = false;
+ }
+
+ public RepairUnsortedFileCompactionTask(
+ long timePartition,
+ TsFileManager tsFileManager,
+ TsFileResource sourceFile,
+ boolean sequence,
+ boolean rewriteFile,
+ long serialId) {
+ super(
+ timePartition,
+ tsFileManager,
+ Collections.singletonList(sourceFile),
+ sequence,
+ new RepairUnsortedFileCompactionPerformer(rewriteFile),
+ serialId,
+ CompactionTaskPriorityType.REPAIR_DATA);
+ this.sourceFile = sourceFile;
+ if (rewriteFile) {
+ this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator();
+ }
+ this.rewriteFile = rewriteFile;
+ }
+
+ public RepairUnsortedFileCompactionTask(
+ long timePartition,
+ TsFileManager tsFileManager,
+ TsFileResource sourceFile,
+ CountDownLatch latch,
+ boolean sequence,
+ long serialId) {
+ super(
+ timePartition,
+ tsFileManager,
+ Collections.singletonList(sourceFile),
+ sequence,
+ new RepairUnsortedFileCompactionPerformer(true),
+ serialId,
+ CompactionTaskPriorityType.REPAIR_DATA);
+ this.sourceFile = sourceFile;
+ this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator();
+ this.latch = latch;
+ this.rewriteFile = false;
+ }
+
+ public RepairUnsortedFileCompactionTask(
+ long timePartition,
+ TsFileManager tsFileManager,
+ TsFileResource sourceFile,
+ CountDownLatch latch,
+ boolean sequence,
+ boolean rewriteFile,
+ long serialId) {
+ super(
+ timePartition,
+ tsFileManager,
+ Collections.singletonList(sourceFile),
+ sequence,
+ new RepairUnsortedFileCompactionPerformer(rewriteFile),
+ serialId,
+ CompactionTaskPriorityType.REPAIR_DATA);
+ this.sourceFile = sourceFile;
+ if (rewriteFile) {
+ this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator();
+ }
+ this.rewriteFile = rewriteFile;
+ this.latch = latch;
+ }
+
+ @Override
+ protected void prepare() throws IOException {
+ targetTsFileResource =
+ new TsFileResource(generateTargetFile(), TsFileResourceStatus.COMPACTING);
+ String dataDirectory = sourceFile.getTsFile().getParent();
+ logFile =
+ new File(
+ dataDirectory
+ + File.separator
+ + targetTsFileResource.getTsFile().getName()
+ + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX);
+ }
+
+ private File generateTargetFile() throws IOException {
+ String path = sourceFile.getTsFile().getParentFile().getPath();
+ // if source file is sequence, the sequence data path should be replaced to unsequence
+ if (sourceFile.isSeq()) {
+ int pos = path.lastIndexOf("sequence");
+ path = path.substring(0, pos) + "unsequence" + path.substring(pos + "sequence".length());
+ }
+
+ TsFileNameGenerator.TsFileName tsFileName =
+ TsFileNameGenerator.getTsFileName(sourceFile.getTsFile().getName());
+ tsFileName.setInnerCompactionCnt(tsFileName.getInnerCompactionCnt() + 1);
+ String fileNameStr =
+ String.format(
+ "%d-%d-%d-%d.tsfile",
+ tsFileName.getTime(), tsFileName.getVersion(), tsFileName.getInnerCompactionCnt(), 0);
+ File targetTsFile = new File(path + File.separator + fileNameStr);
+ if (!targetTsFile.getParentFile().exists()) {
+ targetTsFile.getParentFile().mkdirs();
+ }
+ return targetTsFile;
+ }
+
+ @Override
+ protected void prepareTargetFiles() throws IOException {
+ CompactionUtils.updateProgressIndex(
+ targetTsFileList, selectedTsFileResourceList, Collections.emptyList());
+ CompactionUtils.moveTargetFile(targetTsFileList, true, storageGroupName + "-" + dataRegionId);
+
+ LOGGER.info(
+ "{}-{} [InnerSpaceCompactionTask] start to rename mods file",
+ storageGroupName,
+ dataRegionId);
+
+ if (rewriteFile) {
+ CompactionUtils.combineModsInInnerCompaction(
+ selectedTsFileResourceList, targetTsFileResource);
+ } else {
+ if (sourceFile.modFileExists()) {
+ Files.createLink(
+ new File(targetTsFileResource.getModFile().getFilePath()).toPath(),
+ new File(sourceFile.getModFile().getFilePath()).toPath());
+ }
+ }
+ }
+
+ @Override
+ protected boolean doCompaction() {
+ boolean isSuccess = super.doCompaction();
+ if (!isSuccess) {
+ LOGGER.info("Failed to repair file {}", sourceFile.getTsFile().getAbsolutePath());
+ sourceFile.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ }
+ return isSuccess;
+ }
+
+ @Override
+ public long getEstimatedMemoryCost() {
+ if (innerSpaceEstimator != null && memoryCost == 0L) {
+ try {
+ memoryCost = innerSpaceEstimator.estimateInnerCompactionMemory(selectedTsFileResourceList);
+ } catch (IOException e) {
+ innerSpaceEstimator.cleanup();
+ }
+ }
+ if (memoryCost > SystemInfo.getInstance().getMemorySizeForCompaction()) {
+ sourceFile.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ LOGGER.warn(
+ "[RepairUnsortedFileCompactionTask]"
+ + " Can not repair unsorted file {} "
+ + "because the required memory to repair"
+ + " is greater than the total compaction memory budget",
+ sourceFile.getTsFile().getAbsolutePath());
+ }
+ return memoryCost;
+ }
+
+ @Override
+ public boolean isDiskSpaceCheckPassed() {
+ if (!rewriteFile) {
+ return true;
+ }
+ return super.isDiskSpaceCheckPassed();
+ }
+
+ @Override
+ public void handleTaskCleanup() {
+ super.handleTaskCleanup();
+ if (latch != null) {
+ latch.countDown();
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
index 289271b0afd..0bd9e5ede89 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
@@ -31,8 +31,7 @@ import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.block.column.Column;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
@@ -117,8 +116,7 @@ public abstract class AbstractCompactionWriter implements AutoCloseable {
public abstract void write(TimeValuePair timeValuePair, int subTaskId) throws IOException;
- public abstract void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException;
+ public abstract void write(TsBlock tsBlock, int subTaskId) throws IOException;
public abstract void endFile() throws IOException;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java
index f601aad5301..6bc3a1af406 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCrossCompactionWriter.java
@@ -28,8 +28,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.common.block.column.Column;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import java.io.IOException;
@@ -141,8 +140,7 @@ public abstract class AbstractCrossCompactionWriter extends AbstractCompactionWr
/** Write data in batch, only used for aligned device. */
@Override
- public abstract void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException;
+ public abstract void write(TsBlock tsBlock, int subTaskId) throws IOException;
@Override
public void endFile() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java
index 44144fe2cd0..d3d8cf00b1b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractInnerCompactionWriter.java
@@ -27,8 +27,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.read.TimeValuePair;
-import org.apache.iotdb.tsfile.read.common.block.column.Column;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import java.io.IOException;
@@ -92,8 +91,7 @@ public abstract class AbstractInnerCompactionWriter extends AbstractCompactionWr
}
@Override
- public abstract void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException;
+ public abstract void write(TsBlock tsBlock, int subTaskId) throws IOException;
@Override
public void endFile() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
index 348316de785..442004daf92 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java
@@ -26,8 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.block.column.Column;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
@@ -50,8 +49,7 @@ public class FastCrossCompactionWriter extends AbstractCrossCompactionWriter {
}
@Override
- public void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException {
+ public void write(TsBlock tsBlock, int subTaskId) throws IOException {
throw new RuntimeException("Does not support this method in FastCrossCompactionWriter");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
index 45ef004315d..1a39e29efa7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java
@@ -25,8 +25,7 @@ import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.block.column.Column;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
@@ -41,8 +40,7 @@ public class FastInnerCompactionWriter extends AbstractInnerCompactionWriter {
}
@Override
- public void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException {
+ public void write(TsBlock tsBlock, int subTaskId) throws IOException {
throw new RuntimeException("Does not support this method in FastInnerCompactionWriter");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java
index 888be3f0f07..12cb1196b88 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointCrossCompactionWriter.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.column.Column;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
@@ -43,8 +44,10 @@ public class ReadPointCrossCompactionWriter extends AbstractCrossCompactionWrite
}
@Override
- public void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException {
+ public void write(TsBlock tsBlock, int subTaskId) throws IOException {
+ TimeColumn timestamps = tsBlock.getTimeColumn();
+ Column[] columns = tsBlock.getValueColumns();
+ int batchSize = tsBlock.getPositionCount();
// Since a batch of data is written, the end time of the current aligned device may exceed the
// end time of the device in the source file, but no error will be caused
checkTimeAndMayFlushChunkToCurrentFile(timestamps.getStartTime(), subTaskId);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointInnerCompactionWriter.java
index e84510e1cd2..1730f3547eb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointInnerCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/ReadPointInnerCompactionWriter.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.file.header.PageHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.column.Column;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
@@ -38,8 +39,10 @@ public class ReadPointInnerCompactionWriter extends AbstractInnerCompactionWrite
}
@Override
- public void write(TimeColumn timestamps, Column[] columns, int subTaskId, int batchSize)
- throws IOException {
+ public void write(TsBlock tsBlock, int subTaskId) throws IOException {
+ TimeColumn timestamps = tsBlock.getTimeColumn();
+ Column[] columns = tsBlock.getValueColumns();
+ int batchSize = tsBlock.getPositionCount();
AlignedChunkWriterImpl chunkWriter = (AlignedChunkWriterImpl) this.chunkWriters[subTaskId];
chunkWriter.write(timestamps, columns, batchSize);
chunkPointNumArray[subTaskId] += timestamps.getTimes().length;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java
new file mode 100644
index 00000000000..f7bb83fe922
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/RepairUnsortedFileCompactionWriter.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer;
+
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.reader.IPointReader;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
+import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+/** Used for fixing files which contains internal unsorted data */
+public class RepairUnsortedFileCompactionWriter extends ReadPointInnerCompactionWriter {
+ private List<TimeValuePair> dataOfCurrentSeries;
+
+ public RepairUnsortedFileCompactionWriter(TsFileResource targetFileResource) throws IOException {
+ super(targetFileResource);
+ }
+
+ @Override
+ public void startMeasurement(List<IMeasurementSchema> measurementSchemaList, int subTaskId) {
+ dataOfCurrentSeries = new ArrayList<>();
+ super.startMeasurement(measurementSchemaList, subTaskId);
+ }
+
+ @Override
+ public void endMeasurement(int subTaskId) throws IOException {
+ if (dataOfCurrentSeries.isEmpty()) {
+ super.endMeasurement(subTaskId);
+ return;
+ }
+ // remove duplicate timestamp and sort
+ dataOfCurrentSeries.sort(Comparator.comparingLong(TimeValuePair::getTimestamp));
+ TimeValuePair previousTimeValuePair = dataOfCurrentSeries.get(0);
+ for (TimeValuePair timeValuePair : dataOfCurrentSeries) {
+ if (timeValuePair.getTimestamp() == previousTimeValuePair.getTimestamp()) {
+ // merge to previous TimeValuePair if is aligned series
+ mergeTimeValuePair(timeValuePair, previousTimeValuePair);
+ continue;
+ }
+ writeDataPoint(
+ previousTimeValuePair.getTimestamp(),
+ previousTimeValuePair.getValue(),
+ chunkWriters[subTaskId]);
+ chunkPointNumArray[subTaskId]++;
+ previousTimeValuePair = timeValuePair;
+ }
+ // write last time value pair
+ writeDataPoint(
+ previousTimeValuePair.getTimestamp(),
+ previousTimeValuePair.getValue(),
+ chunkWriters[subTaskId]);
+ chunkPointNumArray[subTaskId]++;
+
+ dataOfCurrentSeries = null;
+ super.endMeasurement(subTaskId);
+ }
+
+ private void mergeTimeValuePair(TimeValuePair from, TimeValuePair to) {
+ if (!isAlign) {
+ // ignore not aligned TimeValuePair with same timestamp
+ return;
+ }
+ TsPrimitiveType[] fromVector = from.getValue().getVector();
+ TsPrimitiveType[] toVector = to.getValue().getVector();
+ for (int i = 0; i < toVector.length; i++) {
+ if (toVector[i] == null && fromVector[i] != null) {
+ toVector[i] = fromVector[i];
+ }
+ }
+ to.getValue().setVector(toVector);
+ }
+
+ @Override
+ public void write(TsBlock tsBlock, int subTaskId) throws IOException {
+ IPointReader pointReader = tsBlock.getTsBlockAlignedRowIterator();
+ while (pointReader.hasNextTimeValuePair()) {
+ write(pointReader.nextTimeValuePair(), subTaskId);
+ }
+ }
+
+ @Override
+ public void write(TimeValuePair timeValuePair, int subTaskId) throws IOException {
+ dataOfCurrentSeries.add(timeValuePair);
+ isEmptyFile = false;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLogger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLogger.java
new file mode 100644
index 00000000000..e7faaa848f4
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLogger.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class RepairLogger implements Closeable {
+
+ static final String repairTimePartitionStartLogPrefix = "START_TIME_PARTITION";
+ static final String cannotRepairFileLogPrefix = "TSFILE";
+ static final String repairTimePartitionEndLogPrefix = "END_TIME_PARTITION";
+ public static final String repairLogSuffix = ".repair-data.log";
+ public static final String repairLogDir = "repair";
+ private final File logFile;
+ private final long repairTaskStartTime;
+ private final FileOutputStream logStream;
+
+ public RepairLogger() throws IOException {
+ this.repairTaskStartTime = System.currentTimeMillis();
+ File logFileDir =
+ new File(
+ IoTDBDescriptor.getInstance().getConfig().getSystemDir()
+ + File.separator
+ + repairLogDir);
+ if (!logFileDir.exists()) {
+ logFileDir.mkdirs();
+ }
+ File[] files = logFileDir.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.getName().endsWith(repairLogSuffix)) {
+ Files.delete(file.toPath());
+ }
+ }
+ }
+ String logFileName = String.format("%s%s", repairTaskStartTime, repairLogSuffix);
+ this.logFile = new File(logFileDir.getPath() + File.separator + logFileName);
+ Path logFilePath = logFile.toPath();
+ if (!Files.exists(logFilePath)) {
+ Files.createFile(logFilePath);
+ }
+ this.logStream = new FileOutputStream(logFile);
+ }
+
+ public RepairLogger(File logFile) throws FileNotFoundException {
+ this.logFile = logFile;
+ String logFileName = logFile.getName();
+ this.repairTaskStartTime = Long.parseLong(logFileName.replace(repairLogSuffix, ""));
+ this.logStream = new FileOutputStream(logFile, true);
+ }
+
+ public void recordRepairedTimePartition(RepairTimePartition timePartition) throws IOException {
+ markStartOfRepairedTimePartition(timePartition);
+ recordCannotRepairFiles(timePartition);
+ markEndOfRepairedTimePartition(timePartition);
+ }
+
+ public void recordCannotRepairFiles(RepairTimePartition timePartition) throws IOException {
+ List<TsFileResource> resources = timePartition.getAllFileSnapshot();
+ List<TsFileResource> cannotRepairFiles =
+ resources.stream()
+ .filter(
+ resource -> resource.getTsFileRepairStatus() == TsFileRepairStatus.CAN_NOT_REPAIR)
+ .collect(Collectors.toList());
+ for (TsFileResource cannotRepairFile : cannotRepairFiles) {
+ recordOneFile(cannotRepairFile);
+ }
+ }
+
+ public void markStartOfRepairedTimePartition(RepairTimePartition timePartition)
+ throws IOException {
+ String startTimePartitionLog =
+ String.format(
+ "%s %s %s %s\n",
+ repairTimePartitionStartLogPrefix,
+ timePartition.getDatabaseName(),
+ timePartition.getDataRegionId(),
+ timePartition.getTimePartitionId());
+ logStream.write(startTimePartitionLog.getBytes());
+ }
+
+ public void markEndOfRepairedTimePartition(RepairTimePartition timePartition) throws IOException {
+ String endTimePartitionLog = String.format("%s\n", repairTimePartitionEndLogPrefix);
+ logStream.write(endTimePartitionLog.getBytes());
+ logStream.flush();
+ }
+
+ public void recordOneFile(TsFileResource resource) throws IOException {
+ String fileLog =
+ String.format("%s %s\n", cannotRepairFileLogPrefix, resource.getTsFile().getName());
+ logStream.write(fileLog.getBytes());
+ }
+
+ public String getRepairLogFilePath() {
+ return logFile.getAbsolutePath();
+ }
+
+ public long getRepairTaskStartTime() {
+ return repairTaskStartTime;
+ }
+
+ @Override
+ public void close() throws IOException {
+ logStream.getFD().sync();
+ logStream.close();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTaskRecoverLogParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTaskRecoverLogParser.java
new file mode 100644
index 00000000000..19dfee8b0d5
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTaskRecoverLogParser.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class RepairTaskRecoverLogParser {
+
+ private final File logFile;
+ private static final Logger LOGGER = LoggerFactory.getLogger(RepairTaskRecoverLogParser.class);
+ private final Map<RepairTimePartition, Set<String>> repairedTimePartitionsWithCannotRepairFiles =
+ new HashMap<>();
+ private RepairTimePartition currentTimePartition;
+ private Set<String> currentTimePartitionCannotRepairFiles;
+
+ public RepairTaskRecoverLogParser(File logFile) {
+ this.logFile = logFile;
+ }
+
+ void parse() throws IOException {
+ parseLogFile();
+ }
+
+ private void parseLogFile() throws IOException {
+ try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) {
+ String curLine;
+ while ((curLine = reader.readLine()) != null) {
+ if (curLine.startsWith(RepairLogger.repairTimePartitionStartLogPrefix)) {
+ parseStartTimePartitionLog(curLine.trim());
+ } else if (curLine.startsWith(RepairLogger.repairTimePartitionEndLogPrefix)) {
+ parseEndTimePartitionLog(curLine.trim());
+ } else if (curLine.startsWith(RepairLogger.cannotRepairFileLogPrefix)) {
+ parseFileLog(curLine.trim());
+ } else {
+ throw new IllegalArgumentException("Unknown format of repair log");
+ }
+ }
+ if (currentTimePartition != null) {
+ handleIncompleteRepairedTimePartition();
+ }
+ }
+ }
+
+ private void parseStartTimePartitionLog(String line) {
+ if (currentTimePartition != null) {
+ handleIncompleteRepairedTimePartition();
+ }
+ String[] values = line.split(" ");
+ if (values.length != 4) {
+ throw new RuntimeException(String.format("String '%s' is not a legal repair log", line));
+ }
+ currentTimePartition = new RepairTimePartition(values[1], values[2], Long.parseLong(values[3]));
+ currentTimePartitionCannotRepairFiles = new HashSet<>();
+ }
+
+ private void parseFileLog(String line) {
+ String[] values = line.split(" ");
+ if (values.length != 2) {
+ throw new RuntimeException(String.format("String '%s' is not a legal repair log", line));
+ }
+ currentTimePartitionCannotRepairFiles.add(values[1]);
+ }
+
+ private void parseEndTimePartitionLog(String line) {
+ repairedTimePartitionsWithCannotRepairFiles.put(
+ currentTimePartition, currentTimePartitionCannotRepairFiles);
+ currentTimePartition = null;
+ currentTimePartitionCannotRepairFiles = null;
+ }
+
+ private void handleIncompleteRepairedTimePartition() {
+ LOGGER.error(
+ "[{}][{}]Repair data log is not complete, time partition is {}.",
+ currentTimePartition.getDatabaseName(),
+ currentTimePartition.getDataRegionId(),
+ currentTimePartition.getTimePartitionId());
+ repairedTimePartitionsWithCannotRepairFiles.put(
+ currentTimePartition, currentTimePartitionCannotRepairFiles);
+ }
+
+ Map<RepairTimePartition, Set<String>> getRepairedTimePartitionsWithCannotRepairFiles() {
+ return repairedTimePartitionsWithCannotRepairFiles;
+ }
+
+ String getRepairLogFilePath() {
+ return logFile.getAbsolutePath();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartition.java
new file mode 100644
index 00000000000..3afb398c280
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairTimePartition.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;
+
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class RepairTimePartition {
+ private final String databaseName;
+ private final String dataRegionId;
+ private TsFileManager tsFileManager;
+ private final long timePartitionId;
+ private long maxFileTimestamp;
+ private boolean repaired;
+
+ public RepairTimePartition(DataRegion dataRegion, long timePartitionId, long maxFileTimestamp) {
+ this.databaseName = dataRegion.getDatabaseName();
+ this.dataRegionId = dataRegion.getDataRegionId();
+ this.tsFileManager = dataRegion.getTsFileManager();
+ this.timePartitionId = timePartitionId;
+ this.maxFileTimestamp = maxFileTimestamp;
+ this.repaired = false;
+ }
+
+ public RepairTimePartition(String databaseName, String dataRegionId, long timePartitionId) {
+ this.databaseName = databaseName;
+ this.dataRegionId = dataRegionId;
+ this.timePartitionId = timePartitionId;
+ }
+
+ public long getTimePartitionId() {
+ return timePartitionId;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String getDataRegionId() {
+ return dataRegionId;
+ }
+
+ public TsFileManager getTsFileManager() {
+ return tsFileManager;
+ }
+
+ public boolean isRepaired() {
+ return repaired;
+ }
+
+ public void setRepaired(boolean repaired) {
+ this.repaired = repaired;
+ }
+
+ public List<TsFileResource> getSeqFileSnapshot() {
+ return tsFileManager.getTsFileListSnapshot(timePartitionId, true).stream()
+ .filter(this::resourceTimestampFilter)
+ .collect(Collectors.toList());
+ }
+
+ public List<TsFileResource> getUnSeqFileSnapshot() {
+ return tsFileManager.getTsFileListSnapshot(timePartitionId, false).stream()
+ .filter(this::resourceTimestampFilter)
+ .collect(Collectors.toList());
+ }
+
+ public List<TsFileResource> getAllFileSnapshot() {
+ List<TsFileResource> seqFiles = getSeqFileSnapshot();
+ List<TsFileResource> unseqFiles = getUnSeqFileSnapshot();
+ List<TsFileResource> allFiles = new ArrayList<>(seqFiles.size() + unseqFiles.size());
+ allFiles.addAll(seqFiles);
+ allFiles.addAll(unseqFiles);
+ return allFiles;
+ }
+
+ private boolean resourceTimestampFilter(TsFileResource resource) {
+ if (resource.getStatus() == TsFileResourceStatus.DELETED
+ || resource.getStatus() == TsFileResourceStatus.UNCLOSED) {
+ return false;
+ }
+ long fileTimestamp = getFileTimestamp(resource);
+ return fileTimestamp >= 0 && fileTimestamp <= maxFileTimestamp;
+ }
+
+ private long getFileTimestamp(TsFileResource resource) {
+ long timestamp = -1;
+ try {
+ TsFileNameGenerator.TsFileName tsFileName =
+ TsFileNameGenerator.getTsFileName(resource.getTsFile().getName());
+ timestamp = tsFileName.getTime();
+ } catch (IOException ignored) {
+ }
+ return timestamp;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ RepairTimePartition that = (RepairTimePartition) o;
+ return timePartitionId == that.timePartitionId
+ && Objects.equals(databaseName, that.databaseName)
+ && Objects.equals(dataRegionId, that.dataRegionId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(databaseName, dataRegionId, timePartitionId);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java
new file mode 100644
index 00000000000..6a480e8ad1a
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;
+
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduler;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class UnsortedFileRepairTaskScheduler implements Runnable {
+
+ /** a repair task is running */
+ private static final AtomicBoolean isRepairingData = new AtomicBoolean(false);
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(UnsortedFileRepairTaskScheduler.class);
+ private final Set<RepairTimePartition> allTimePartitionFiles = new HashSet<>();
+ private RepairLogger repairLogger;
+ private final boolean isRecover;
+ private boolean initSuccess = false;
+ private long repairTaskTime;
+ private int repairProgress = 0;
+
+ public static boolean markRepairTaskStart() {
+ return isRepairingData.compareAndSet(false, true);
+ }
+
+ public static void markRepairTaskFinish() {
+ isRepairingData.set(false);
+ }
+
+ /** Used for create a new repair schedule task */
+ public UnsortedFileRepairTaskScheduler(List<DataRegion> dataRegions) {
+ this.isRecover = false;
+ try {
+ repairLogger = new RepairLogger();
+ } catch (Exception e) {
+ try {
+ LOGGER.error("[RepairScheduler] Failed to create repair logger", e);
+ repairLogger.close();
+ } catch (IOException closeException) {
+ LOGGER.error("[RepairScheduler] Failed to close repair logger", closeException);
+ }
+ return;
+ }
+ this.repairTaskTime = repairLogger.getRepairTaskStartTime();
+ collectTimePartitions(dataRegions);
+ initSuccess = true;
+ }
+
+ /** Used for recover from log file */
+ public UnsortedFileRepairTaskScheduler(List<DataRegion> dataRegions, File logFile) {
+ this.isRecover = true;
+ LOGGER.info("[RepairScheduler] start recover repair log {}", logFile.getAbsolutePath());
+ try {
+ repairLogger = new RepairLogger(logFile);
+ } catch (Exception e) {
+ try {
+ LOGGER.error(
+ "[RepairScheduler] Failed to get repair logger from log file {}",
+ logFile.getAbsolutePath(),
+ e);
+ repairLogger.close();
+ } catch (IOException closeException) {
+ LOGGER.error(
+ "[RepairScheduler] Failed to close log file {}",
+ logFile.getAbsolutePath(),
+ closeException);
+ }
+ return;
+ }
+ this.repairTaskTime = repairLogger.getRepairTaskStartTime();
+ collectTimePartitions(dataRegions);
+ try {
+ recover(logFile);
+ } catch (Exception e) {
+ LOGGER.error(
+ "[RepairScheduler] Failed to parse repair log file {}", logFile.getAbsolutePath(), e);
+ return;
+ }
+ initSuccess = true;
+ }
+
+ private void recover(File logFile) throws IOException {
+ RepairTaskRecoverLogParser recoverLogParser = new RepairTaskRecoverLogParser(logFile);
+ LOGGER.info(
+ "[RepairScheduler] recover unfinished repair schedule task from log file: {}",
+ recoverLogParser.getRepairLogFilePath());
+ recoverLogParser.parse();
+ Map<RepairTimePartition, Set<String>> repairedTimePartitionWithCannotRepairFiles =
+ recoverLogParser.getRepairedTimePartitionsWithCannotRepairFiles();
+ for (RepairTimePartition timePartition : allTimePartitionFiles) {
+ Set<String> cannotRepairFiles =
+ repairedTimePartitionWithCannotRepairFiles.remove(timePartition);
+ if (cannotRepairFiles == null) {
+ continue;
+ }
+ // mark time partition as repaired
+ timePartition.setRepaired(true);
+ if (cannotRepairFiles.isEmpty()) {
+ continue;
+ }
+ // mark cannot repair file in TsFileResource
+ List<TsFileResource> resources = timePartition.getAllFileSnapshot();
+ for (TsFileResource resource : resources) {
+ if (resource.getStatus() != TsFileResourceStatus.NORMAL) {
+ continue;
+ }
+ if (cannotRepairFiles.contains(resource.getTsFile().getName())) {
+ resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR);
+ }
+ }
+ }
+ }
+
+ private void collectTimePartitions(List<DataRegion> dataRegions) {
+ for (DataRegion dataRegion : dataRegions) {
+ if (dataRegion == null) {
+ continue;
+ }
+ List<Long> timePartitions = dataRegion.getTimePartitions();
+ for (long timePartition : timePartitions) {
+ allTimePartitionFiles.add(
+ new RepairTimePartition(dataRegion, timePartition, repairTaskTime));
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ if (!initSuccess) {
+ LOGGER.info("[RepairScheduler] Failed to init repair schedule task");
+ markRepairTaskFinish();
+ return;
+ }
+ CompactionScheduler.exclusiveLockCompactionSelection();
+ CompactionTaskManager.getInstance().waitAllCompactionFinish();
+ try {
+ executeRepair();
+ } catch (Exception e) {
+ LOGGER.error("[RepairScheduler] Meet error when execute repair schedule task", e);
+ } finally {
+ markRepairTaskFinish();
+ try {
+ repairLogger.close();
+ } catch (Exception e) {
+ LOGGER.error(
+ "[RepairScheduler] Failed to close repair logger {}",
+ repairLogger.getRepairLogFilePath(),
+ e);
+ }
+ LOGGER.info("[RepairScheduler] Finished repair task");
+ CompactionScheduler.exclusiveUnlockCompactionSelection();
+ }
+ }
+
+ private void executeRepair() throws InterruptedException {
+ for (RepairTimePartition timePartition : allTimePartitionFiles) {
+ if (timePartition.isRepaired()) {
+ LOGGER.info(
+ "[RepairScheduler][{}][{}] skip repair time partition {} because it is repaired",
+ timePartition.getDatabaseName(),
+ timePartition.getDataRegionId(),
+ timePartition.getTimePartitionId());
+ repairProgress++;
+ continue;
+ }
+ // repair unsorted data in single file
+ checkInternalUnsortedFileAndRepair(timePartition);
+ // repair unsorted data between sequence files
+ checkOverlapInSequenceSpaceAndRepair(timePartition);
+ finishRepairTimePartition(timePartition);
+ }
+ }
+
+ private void checkInternalUnsortedFileAndRepair(RepairTimePartition timePartition)
+ throws InterruptedException {
+ List<TsFileResource> sourceFiles =
+ Stream.concat(
+ timePartition.getSeqFileSnapshot().stream(),
+ timePartition.getUnSeqFileSnapshot().stream())
+ .collect(Collectors.toList());
+ for (TsFileResource sourceFile : sourceFiles) {
+ sourceFile.readLock();
+ try {
+ if (sourceFile.getStatus() != TsFileResourceStatus.NORMAL) {
+ continue;
+ }
+ if (TsFileResourceUtils.validateTsFileDataCorrectness(sourceFile)) {
+ continue;
+ }
+ } finally {
+ sourceFile.readUnlock();
+ }
+ LOGGER.info(
+ "[RepairScheduler] file {} need to repair because it has internal unsorted data",
+ sourceFile);
+ TsFileManager tsFileManager = timePartition.getTsFileManager();
+ CountDownLatch latch = new CountDownLatch(1);
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(
+ timePartition.getTimePartitionId(),
+ timePartition.getTsFileManager(),
+ sourceFile,
+ latch,
+ sourceFile.isSeq(),
+ tsFileManager.getNextCompactionTaskId());
+ if (CompactionTaskManager.getInstance().addTaskToWaitingQueue(task)) {
+ latch.await();
+ }
+ }
+ }
+
+ private void checkOverlapInSequenceSpaceAndRepair(RepairTimePartition timePartition)
+ throws InterruptedException {
+ TsFileManager tsFileManager = timePartition.getTsFileManager();
+ List<TsFileResource> seqList =
+ tsFileManager.getTsFileListSnapshot(timePartition.getTimePartitionId(), true);
+ List<TsFileResource> overlapFiles = checkTimePartitionHasOverlap(seqList);
+ for (TsFileResource overlapFile : overlapFiles) {
+ CountDownLatch latch = new CountDownLatch(1);
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(
+ timePartition.getTimePartitionId(),
+ timePartition.getTsFileManager(),
+ overlapFile,
+ latch,
+ true,
+ false,
+ tsFileManager.getNextCompactionTaskId());
+ LOGGER.info(
+ "[RepairScheduler] file {} need to repair because it is overlapped with other files",
+ overlapFile);
+ if (CompactionTaskManager.getInstance().addTaskToWaitingQueue(task)) {
+ latch.await();
+ }
+ }
+ }
+
+ private List<TsFileResource> checkTimePartitionHasOverlap(List<TsFileResource> resources) {
+ List<TsFileResource> overlapResources = new ArrayList<>();
+ Map<String, Long> deviceEndTimeMap = new HashMap<>();
+ for (TsFileResource resource : resources) {
+ if (resource.getStatus() == TsFileResourceStatus.UNCLOSED
+ || resource.getStatus() == TsFileResourceStatus.DELETED) {
+ continue;
+ }
+ DeviceTimeIndex deviceTimeIndex;
+ try {
+ deviceTimeIndex = getDeviceTimeIndex(resource);
+ } catch (Exception ignored) {
+ continue;
+ }
+
+ Set<String> devices = deviceTimeIndex.getDevices();
+ boolean fileHasOverlap = false;
+ // check overlap
+ for (String device : devices) {
+ long deviceStartTimeInCurrentFile = deviceTimeIndex.getStartTime(device);
+ if (deviceStartTimeInCurrentFile > deviceTimeIndex.getEndTime(device)) {
+ continue;
+ }
+ if (!deviceEndTimeMap.containsKey(device)) {
+ continue;
+ }
+ long deviceEndTimeInPreviousFile = deviceEndTimeMap.get(device);
+ if (deviceStartTimeInCurrentFile <= deviceEndTimeInPreviousFile) {
+ fileHasOverlap = true;
+ overlapResources.add(resource);
+ break;
+ }
+ }
+ // update end time map
+ if (!fileHasOverlap) {
+ for (String device : devices) {
+ deviceEndTimeMap.put(device, deviceTimeIndex.getEndTime(device));
+ }
+ }
+ }
+ return overlapResources;
+ }
+
+ private DeviceTimeIndex getDeviceTimeIndex(TsFileResource resource) throws IOException {
+ ITimeIndex timeIndex = resource.getTimeIndex();
+ if (timeIndex instanceof DeviceTimeIndex) {
+ return (DeviceTimeIndex) timeIndex;
+ }
+ return resource.buildDeviceTimeIndex();
+ }
+
+ private void finishRepairTimePartition(RepairTimePartition timePartition) {
+ try {
+ repairLogger.recordRepairedTimePartition(timePartition);
+ } catch (Exception e) {
+ LOGGER.error(
+ "[RepairScheduler][{}][{}] failed to record repair log for time partition {}",
+ timePartition.getDatabaseName(),
+ timePartition.getDataRegionId(),
+ timePartition.getTimePartitionId());
+ }
+ repairProgress++;
+ LOGGER.info(
+ "[RepairScheduler][{}][{}] time partition {} has been repaired, progress: {}/{}",
+ timePartition.getDatabaseName(),
+ timePartition.getDataRegionId(),
+ timePartition.getTimePartitionId(),
+ repairProgress,
+ allTimePartitionFiles.size());
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
index e8ee18fd5fa..a168c31a273 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
@@ -40,8 +40,8 @@ import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.Phaser;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
/**
@@ -62,14 +62,22 @@ public class CompactionScheduler {
private CompactionScheduler() {}
/** avoid timed compaction schedule conflict with manually triggered schedule */
- private static final Lock compactionTaskSelectionLock = new ReentrantLock();
+ private static final ReadWriteLock compactionTaskSelectionLock = new ReentrantReadWriteLock();
- public static void lockCompactionSelection() {
- compactionTaskSelectionLock.lock();
+ public static void sharedLockCompactionSelection() {
+ compactionTaskSelectionLock.readLock().lock();
}
- public static void unlockCompactionSelection() {
- compactionTaskSelectionLock.unlock();
+ public static void sharedUnlockCompactionSelection() {
+ compactionTaskSelectionLock.readLock().unlock();
+ }
+
+ public static void exclusiveLockCompactionSelection() {
+ compactionTaskSelectionLock.writeLock().lock();
+ }
+
+ public static void exclusiveUnlockCompactionSelection() {
+ compactionTaskSelectionLock.writeLock().unlock();
}
/**
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
index 97c5caa50ec..4cd669a7e54 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskManager.java
@@ -147,7 +147,6 @@ public class CompactionTaskManager implements IService {
}
@SuppressWarnings({"squid:S2142", "squid:S135", "VariableDeclarationUsageDistanceCheck"})
- @TestOnly
public void waitAllCompactionFinish() {
if (taskExecutionPool != null) {
WrappedThreadPoolExecutor tmpThreadPool = taskExecutionPool;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
index 77a4bc934f2..d4353a9807e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.compara
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskPriorityType;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InsertionCrossSpaceCompactionTask;
@@ -74,10 +73,13 @@ public class DefaultCompactionTaskComparatorImpl implements ICompactionTaskCompa
public int compareInnerSpaceCompactionTask(
InnerSpaceCompactionTask o1, InnerSpaceCompactionTask o2) {
- // if compactionTaskType of o1 and o2 are different
- // we prefer to execute task type with MOD_SETTLE
+ // if compactionTaskPriorityType of o1 and o2 are different
+ // we prefer to execute task type with high priority type
if (o1.getCompactionTaskPriorityType() != o2.getCompactionTaskPriorityType()) {
- return o1.getCompactionTaskPriorityType() == CompactionTaskPriorityType.MOD_SETTLE ? -1 : 1;
+ return o2.getCompactionTaskPriorityType().getValue()
+ > o1.getCompactionTaskPriorityType().getValue()
+ ? 1
+ : -1;
}
// if max mods file size of o1 and o2 are different
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
new file mode 100644
index 00000000000..3eadd41ea28
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
+
+public class RepairUnsortedFileCompactionEstimator extends AbstractInnerSpaceEstimator {
+ @Override
+ protected long calculatingMetadataMemoryCost(CompactionTaskInfo taskInfo) {
+ long cost = 0;
+ // add ChunkMetadata size of MultiTsFileDeviceIterator
+ cost +=
+ Math.min(
+ taskInfo.getTotalChunkMetadataSize(),
+ taskInfo.getMaxChunkMetadataNumInDevice() * taskInfo.getMaxChunkMetadataSize());
+
+ // add ChunkMetadata size of targetFileWriter
+ long sizeForFileWriter =
+ (long)
+ ((double) SystemInfo.getInstance().getMemorySizeForCompaction()
+ / IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount()
+ * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
+ cost += sizeForFileWriter;
+
+ return cost;
+ }
+
+ @Override
+ protected long calculatingDataMemoryCost(CompactionTaskInfo taskInfo) {
+ if (taskInfo.getTotalChunkNum() == 0) {
+ return taskInfo.getModificationFileSize();
+ }
+ long maxConcurrentSeriesNum =
+ Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum());
+ long averageUncompressedChunkSize =
+ taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum();
+
+ long maxConcurrentSeriesSize =
+ averageUncompressedChunkSize
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ / compressionRatio;
+ long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum;
+ long targetChunkWriterSize = Math.min(maxConcurrentSeriesSize, maxTargetChunkWriterSize);
+
+ long inMemorySortedDataSize =
+ averageUncompressedChunkSize
+ * Math.min(
+ taskInfo.getMaxChunkMetadataNumInDevice(),
+ taskInfo.getMaxChunkMetadataNumInSeries() * maxConcurrentSeriesNum);
+
+ return targetChunkWriterSize + inMemorySortedDataSize + taskInfo.getModificationFileSize();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
index 23b31ec314e..2dee4544b01 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskPriorityType;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.comparator.ICompactionTaskComparator;
@@ -32,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.IInnerSe
import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.IInnerUnseqSpaceSelector;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
@@ -113,7 +115,7 @@ public class SizeTieredCompactionSelector
selectedFileSize = 0L;
continue;
}
- if (currentFile.getStatus() != TsFileResourceStatus.NORMAL) {
+ if (cannotSelectCurrentFileToNormalCompaction(currentFile)) {
selectedFileList.clear();
selectedFileSize = 0L;
continue;
@@ -161,6 +163,12 @@ public class SizeTieredCompactionSelector
return taskList;
}
+ private boolean cannotSelectCurrentFileToNormalCompaction(TsFileResource resource) {
+ return resource.getStatus() != TsFileResourceStatus.NORMAL
+ || resource.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR
+ || resource.getTsFileRepairStatus() == TsFileRepairStatus.CAN_NOT_REPAIR;
+ }
+
/**
* This method is used to select a batch of files to be merged. There are two ways to select
* files.If the first method selects the appropriate file, the second method is not executed. The
@@ -176,12 +184,17 @@ public class SizeTieredCompactionSelector
public List<InnerSpaceCompactionTask> selectInnerSpaceTask(List<TsFileResource> tsFileResources) {
this.tsFileResources = tsFileResources;
try {
- // 1. preferentially select compaction task based on mod file
- List<InnerSpaceCompactionTask> taskList = selectTaskBaseOnModFile();
+ // 1. select compaction task based on file which need to repair
+ List<InnerSpaceCompactionTask> taskList = selectFileNeedToRepair();
+ if (!taskList.isEmpty()) {
+ return taskList;
+ }
+ // 2. preferentially select compaction task based on mod file
+ taskList = selectTaskBaseOnModFile();
if (!taskList.isEmpty()) {
return taskList;
}
- // 2. if a suitable compaction task is not selected in the first step, select the compaction
+ // 3. if a suitable compaction task is not selected in the first step, select the compaction
// task at the tsFile level
return selectTaskBaseOnLevel();
} catch (Exception e) {
@@ -201,6 +214,23 @@ public class SizeTieredCompactionSelector
return Collections.emptyList();
}
+ private List<InnerSpaceCompactionTask> selectFileNeedToRepair() {
+ List<InnerSpaceCompactionTask> taskList = new ArrayList<>();
+ for (TsFileResource resource : tsFileResources) {
+ if (resource.getStatus() == TsFileResourceStatus.NORMAL
+ && resource.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR) {
+ taskList.add(
+ new RepairUnsortedFileCompactionTask(
+ timePartition,
+ tsFileManager,
+ resource,
+ sequence,
+ tsFileManager.getNextCompactionTaskId()));
+ }
+ }
+ return taskList;
+ }
+
private List<InnerSpaceCompactionTask> selectTaskBaseOnModFile() {
List<InnerSpaceCompactionTask> taskList = new ArrayList<>();
for (TsFileResource tsFileResource : tsFileResources) {
@@ -208,7 +238,7 @@ public class SizeTieredCompactionSelector
if (Objects.isNull(modFile) || !modFile.exists()) {
continue;
}
- if (tsFileResource.getStatus() != TsFileResourceStatus.NORMAL) {
+ if (cannotSelectCurrentFileToNormalCompaction(tsFileResource)) {
continue;
}
if (modFile.getSize() > config.getInnerCompactionTaskSelectionModsFileThreshold()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java
index e50fe16884f..68a0e0c453c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
@@ -49,7 +50,9 @@ public class TsFileResourceCandidate {
this.selected = false;
// although we do the judgement here, the task should be validated before executing because
// the status of file may be changed after the task is submitted to queue
- this.isValidCandidate = tsFileResource.getStatus() == TsFileResourceStatus.NORMAL;
+ this.isValidCandidate =
+ tsFileResource.getStatus() == TsFileResourceStatus.NORMAL
+ && tsFileResource.getTsFileRepairStatus() == TsFileRepairStatus.NORMAL;
}
/**
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java
index 9e1e90441a4..b0124bfbf38 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/settle/SettleRequestHandler.java
@@ -74,7 +74,7 @@ public class SettleRequestHandler {
public TSStatus handleSettleRequest(TSettleReq req) {
List<String> paths = req.getPaths();
- CompactionScheduler.lockCompactionSelection();
+ CompactionScheduler.exclusiveLockCompactionSelection();
try {
SettleRequestContext context = new SettleRequestContext(paths);
TSStatus validationResult = context.validate();
@@ -87,7 +87,7 @@ public class SettleRequestHandler {
List<TsFileResource> selectedTsFileResources = context.getTsFileResourcesByFileNames();
return context.submitCompactionTask(selectedTsFileResources);
} finally {
- CompactionScheduler.unlockCompactionSelection();
+ CompactionScheduler.exclusiveUnlockCompactionSelection();
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
index 3d85e88caac..8619ed77857 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java
@@ -90,6 +90,16 @@ public class TsFileManager {
}
}
+ public List<TsFileResource> getTsFileListSnapshot(long timePartition, boolean sequence) {
+ readLock();
+ try {
+ Map<Long, TsFileResourceList> chosenMap = sequence ? sequenceFiles : unsequenceFiles;
+ return new ArrayList<>(chosenMap.getOrDefault(timePartition, new TsFileResourceList()));
+ } finally {
+ readUnlock();
+ }
+ }
+
public List<TsFileResource> getTsFileList(boolean sequence, long startTime, long endTime) {
// the iteration of ConcurrentSkipListMap is not concurrent secure
// so we must add read lock here
@@ -255,8 +265,7 @@ public class TsFileManager {
List<TsFileResource> seqFileResources,
List<TsFileResource> unseqFileResources,
List<TsFileResource> targetFileResources,
- long timePartition,
- boolean isTargetSequence)
+ long timePartition)
throws IOException {
writeLock("replace");
try {
@@ -270,24 +279,20 @@ public class TsFileManager {
TsFileResourceManager.getInstance().removeTsFileResource(tsFileResource);
}
}
- if (isTargetSequence) {
- // seq inner space compaction or cross space compaction
- for (TsFileResource resource : targetFileResources) {
- if (!resource.isDeleted()) {
- TsFileResourceManager.getInstance().registerSealedTsFileResource(resource);
- sequenceFiles.get(timePartition).keepOrderInsert(resource);
- }
- }
- } else {
- // unseq inner space compaction
- for (TsFileResource resource : targetFileResources) {
- if (!resource.isDeleted()) {
- TsFileResourceManager.getInstance().registerSealedTsFileResource(resource);
- unsequenceFiles.get(timePartition).keepOrderInsert(resource);
+ for (TsFileResource resource : targetFileResources) {
+ if (!resource.isDeleted()) {
+ TsFileResourceManager.getInstance().registerSealedTsFileResource(resource);
+ if (resource.isSeq()) {
+ sequenceFiles
+ .computeIfAbsent(timePartition, t -> new TsFileResourceList())
+ .keepOrderInsert(resource);
+ } else {
+ unsequenceFiles
+ .computeIfAbsent(timePartition, t -> new TsFileResourceList())
+ .keepOrderInsert(resource);
}
}
}
-
} finally {
writeUnlock();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java
similarity index 67%
copy from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java
copy to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java
index 79aea997f58..933551d227b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CompactionTaskPriorityType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java
@@ -17,16 +17,10 @@
* under the License.
*/
-package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
+package org.apache.iotdb.db.storageengine.dataregion.tsfile;
-public enum CompactionTaskPriorityType {
- /** default compaction task type */
+public enum TsFileRepairStatus {
NORMAL,
-
- /**
- * in either of the following situations: 1. the TsFile has .mods file whose size exceeds 50 MB.
- * 2. the TsFile has .mods file and the disk availability rate is lower than the
- * disk_space_warning_threshold.
- */
- MOD_SETTLE
+ NEED_TO_REPAIR,
+ CAN_NOT_REPAIR
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 2baabaf0c6e..fac186a1c9f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -104,6 +104,9 @@ public class TsFileResource {
protected AtomicReference<TsFileResourceStatus> atomicStatus =
new AtomicReference<>(TsFileResourceStatus.UNCLOSED);
+ /** used for check whether this file has internal unsorted data in compaction selection */
+ private TsFileRepairStatus tsFileRepairStatus = TsFileRepairStatus.NORMAL;
+
private TsFileLock tsFileLock = new TsFileLock();
private boolean isSeq;
@@ -296,6 +299,10 @@ public class TsFileResource {
return getModFile().exists();
}
+ public boolean compactionModFileExists() {
+ return getCompactionModFile().exists();
+ }
+
public List<IChunkMetadata> getChunkMetadataList(PartialPath seriesPath) {
return new ArrayList<>(pathToChunkMetadataListMap.get(seriesPath));
}
@@ -654,6 +661,14 @@ public class TsFileResource {
}
}
+ public TsFileRepairStatus getTsFileRepairStatus() {
+ return this.tsFileRepairStatus;
+ }
+
+ public void setTsFileRepairStatus(TsFileRepairStatus fileRepairStatus) {
+ this.tsFileRepairStatus = fileRepairStatus;
+ }
+
public void forceMarkDeleted() {
atomicStatus.set(TsFileResourceStatus.DELETED);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
index 4c63c04c6d4..d9a2ce07c7a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
@@ -239,31 +239,34 @@ public class TsFileNameGenerator {
maxCrossMergeCount = Math.max(tsFileName.crossCompactionCnt, maxCrossMergeCount);
}
// set target resource to COMPACTING until the end of this task
- return sequence
- ? new TsFileResource(
- new File(
- tsFileResources.get(0).getTsFile().getParent(),
- minTime
- + FILE_NAME_SEPARATOR
- + minVersion
- + FILE_NAME_SEPARATOR
- + (maxInnerMergeCount + 1)
- + FILE_NAME_SEPARATOR
- + maxCrossMergeCount
- + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
- TsFileResourceStatus.COMPACTING)
- : new TsFileResource(
- new File(
- tsFileResources.get(0).getTsFile().getParent(),
- maxTime
- + FILE_NAME_SEPARATOR
- + maxVersion
- + FILE_NAME_SEPARATOR
- + (maxInnerMergeCount + 1)
- + FILE_NAME_SEPARATOR
- + maxCrossMergeCount
- + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
- TsFileResourceStatus.COMPACTING);
+ TsFileResource resource =
+ sequence
+ ? new TsFileResource(
+ new File(
+ tsFileResources.get(0).getTsFile().getParent(),
+ minTime
+ + FILE_NAME_SEPARATOR
+ + minVersion
+ + FILE_NAME_SEPARATOR
+ + (maxInnerMergeCount + 1)
+ + FILE_NAME_SEPARATOR
+ + maxCrossMergeCount
+ + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
+ TsFileResourceStatus.COMPACTING)
+ : new TsFileResource(
+ new File(
+ tsFileResources.get(0).getTsFile().getParent(),
+ maxTime
+ + FILE_NAME_SEPARATOR
+ + maxVersion
+ + FILE_NAME_SEPARATOR
+ + (maxInnerMergeCount + 1)
+ + FILE_NAME_SEPARATOR
+ + maxCrossMergeCount
+ + IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX),
+ TsFileResourceStatus.COMPACTING);
+ resource.setSeq(sequence);
+ return resource;
}
public static class TsFileName {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java
index 60f1e414b52..b6cb6340611 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationAndRepairTool.java
@@ -199,12 +199,6 @@ public class TsFileOverlapValidationAndRepairTool {
// update end time map
if (!fileHasOverlap) {
for (String device : devices) {
- long deviceEndTimeInCurrentFile = resource.getEndTime(device);
- if (!deviceLastExistTsFileMap.containsKey(device)) {
- deviceEndTimeMap.put(device, deviceEndTimeInCurrentFile);
- deviceLastExistTsFileMap.put(device, resource);
- continue;
- }
deviceEndTimeMap.put(device, resource.getEndTime(device));
deviceLastExistTsFileMap.put(device, resource);
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index 8d15d8c62d0..f0be24b228e 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -2106,7 +2106,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG + "-" + "0");
CompactionUtils.combineModsInInnerCompaction(sourceFiles, targetResources.get(0));
- tsFileManager.replace(sourceFiles, Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.replace(sourceFiles, Collections.emptyList(), targetResources, 0);
CompactionUtils.deleteTsFilesInDisk(sourceFiles, COMPACTION_TEST_SG + "-" + "0");
targetResources.forEach(x -> x.setStatusForTest(TsFileResourceStatus.NORMAL));
@@ -2226,7 +2226,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
CompactionUtils.combineModsInCrossCompaction(
sourceFiles.getSeqFiles(), sourceFiles.getUnseqFiles(), targetResources);
tsFileManager.replace(
- sourceFiles.getSeqFiles(), sourceFiles.getUnseqFiles(), targetResources, 0, true);
+ sourceFiles.getSeqFiles(), sourceFiles.getUnseqFiles(), targetResources, 0);
// Suppose the read lock of the source file is occupied by other threads, causing the first task
// to get stuck.
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index 67dfb9294cc..5978d878352 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -2106,7 +2106,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG + "-" + "0");
CompactionUtils.combineModsInInnerCompaction(sourceFiles, targetResources.get(0));
- tsFileManager.replace(sourceFiles, Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.replace(sourceFiles, Collections.emptyList(), targetResources, 0);
CompactionUtils.deleteTsFilesInDisk(sourceFiles, COMPACTION_TEST_SG + "-" + "0");
// start selecting files and then start a cross space compaction task
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java
index 5730fa17e52..121664a7e3a 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerCompactionTest.java
@@ -118,6 +118,7 @@ public abstract class InnerCompactionTest {
+ 0
+ ".tsfile"));
TsFileResource tsFileResource = new TsFileResource(file);
+ tsFileResource.setSeq(true);
tsFileResource.setStatusForTest(TsFileResourceStatus.NORMAL);
tsFileResource.updatePlanIndexes((long) i);
seqResources.add(tsFileResource);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLoggerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLoggerTest.java
new file mode 100644
index 00000000000..8ec829d8e3f
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairLoggerTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;
+
+import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+
+public class RepairLoggerTest extends AbstractCompactionTest {
+
+ @Test
+ public void testSimpleReadWriteLogFile() throws IOException {
+ TsFileResource resource1 = createEmptyFileAndResource(true);
+ TsFileResource resource2 = createEmptyFileAndResource(true);
+ resource2.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ RepairTimePartition mockRepairTimePartition = Mockito.mock(RepairTimePartition.class);
+ Mockito.when(mockRepairTimePartition.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockRepairTimePartition.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockRepairTimePartition.getTimePartitionId()).thenReturn(0L);
+ Mockito.when(mockRepairTimePartition.getAllFileSnapshot())
+ .thenReturn(Arrays.asList(resource1, resource2));
+ Path tempDirPath = Files.createTempDirectory("");
+ File logFile =
+ new File(
+ tempDirPath.toString()
+ + File.separator
+ + System.currentTimeMillis()
+ + RepairLogger.repairLogSuffix);
+ try (RepairLogger logger = new RepairLogger(logFile)) {
+ logger.recordRepairedTimePartition(mockRepairTimePartition);
+ }
+ RepairTaskRecoverLogParser logParser = new RepairTaskRecoverLogParser(logFile);
+ logParser.parse();
+ Map<RepairTimePartition, Set<String>> repairedTimePartitionsWithCannotRepairFiles =
+ logParser.getRepairedTimePartitionsWithCannotRepairFiles();
+ Assert.assertEquals(1, repairedTimePartitionsWithCannotRepairFiles.size());
+ for (Map.Entry<RepairTimePartition, Set<String>> entry :
+ repairedTimePartitionsWithCannotRepairFiles.entrySet()) {
+ Assert.assertEquals(
+ mockRepairTimePartition.getDatabaseName(), entry.getKey().getDatabaseName());
+ Assert.assertEquals(
+ mockRepairTimePartition.getDataRegionId(), entry.getKey().getDataRegionId());
+ Assert.assertEquals(
+ mockRepairTimePartition.getTimePartitionId(), entry.getKey().getTimePartitionId());
+ Assert.assertTrue(entry.getValue().contains(resource2.getTsFile().getName()));
+ break;
+ }
+ }
+
+ @Test
+ public void testReadIncompleteLogFile() throws IOException {
+ TsFileResource resource1 = createEmptyFileAndResource(true);
+ TsFileResource resource2 = createEmptyFileAndResource(true);
+ resource2.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ RepairTimePartition mockRepairTimePartition = Mockito.mock(RepairTimePartition.class);
+ Mockito.when(mockRepairTimePartition.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockRepairTimePartition.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockRepairTimePartition.getTimePartitionId()).thenReturn(0L);
+ Mockito.when(mockRepairTimePartition.getAllFileSnapshot())
+ .thenReturn(Arrays.asList(resource1, resource2));
+ Path tempDirPath = Files.createTempDirectory("");
+ File logFile =
+ new File(
+ tempDirPath.toString()
+ + File.separator
+ + System.currentTimeMillis()
+ + RepairLogger.repairLogSuffix);
+ try (RepairLogger logger = new RepairLogger(logFile)) {
+ logger.markStartOfRepairedTimePartition(mockRepairTimePartition);
+ logger.recordCannotRepairFiles(mockRepairTimePartition);
+ }
+ RepairTaskRecoverLogParser logParser = new RepairTaskRecoverLogParser(logFile);
+ logParser.parse();
+ Map<RepairTimePartition, Set<String>> repairedTimePartitionsWithCannotRepairFiles =
+ logParser.getRepairedTimePartitionsWithCannotRepairFiles();
+ Assert.assertEquals(1, repairedTimePartitionsWithCannotRepairFiles.size());
+ for (Map.Entry<RepairTimePartition, Set<String>> entry :
+ repairedTimePartitionsWithCannotRepairFiles.entrySet()) {
+ Assert.assertEquals(
+ mockRepairTimePartition.getDatabaseName(), entry.getKey().getDatabaseName());
+ Assert.assertEquals(
+ mockRepairTimePartition.getDataRegionId(), entry.getKey().getDataRegionId());
+ Assert.assertEquals(
+ mockRepairTimePartition.getTimePartitionId(), entry.getKey().getTimePartitionId());
+ Assert.assertTrue(entry.getValue().contains(resource2.getTsFile().getName()));
+ break;
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
new file mode 100644
index 00000000000..e1a9113bc3d
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java
@@ -0,0 +1,973 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction.repair;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleSummary;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduler;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.reader.IPointReader;
+import org.apache.iotdb.tsfile.read.reader.chunk.AlignedChunkReader;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class RepairUnsortedFileCompactionTest extends AbstractCompactionTest {
+
+ private boolean enableSeqSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableSeqSpaceCompaction();
+ private boolean enableUnSeqSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableUnseqSpaceCompaction();
+ private boolean enableCrossSpaceCompaction =
+ IoTDBDescriptor.getInstance().getConfig().isEnableCrossSpaceCompaction();
+
+ @Before
+ public void setUp()
+ throws IOException, WriteProcessException, MetadataException, InterruptedException {
+ super.setUp();
+ Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
+ IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(true);
+ IoTDBDescriptor.getInstance().getConfig().setEnableUnseqSpaceCompaction(true);
+ IoTDBDescriptor.getInstance().getConfig().setEnableCrossSpaceCompaction(true);
+ }
+
+ @After
+ public void tearDown() throws IOException, StorageEngineException {
+ IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(enableSeqSpaceCompaction);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setEnableUnseqSpaceCompaction(enableUnSeqSpaceCompaction);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setEnableCrossSpaceCompaction(enableCrossSpaceCompaction);
+ super.tearDown();
+ }
+
+ @Test
+ public void testRepairUnsortedDataBetweenPageWithNonAlignedSeries() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(5, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource));
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0);
+ task.start();
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileDataCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ }
+
+ @Test
+ public void testRepairUnsortedDataBetweenPageWithAlignedSeries() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(5, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource));
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0);
+ task.start();
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileDataCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ }
+
+ @Test
+ public void testRepairUnsortedDataInOnePageWithNonAlignedSeries() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][][] {
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(10, 20), new TimeRange(29, 30), new TimeRange(21, 25)}
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource));
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0);
+ task.start();
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileDataCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ }
+
+ @Test
+ public void testRepairUnsortedDataInOnePageWithUnseqFile() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(false);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][][] {
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(10, 20), new TimeRange(29, 30), new TimeRange(21, 25)}
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource));
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0);
+ task.start();
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileDataCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ }
+
+ @Test
+ public void testRepairUnsortedDataInOnePageWithAlignedSeries() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][][] {
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(10, 20), new TimeRange(29, 30), new TimeRange(21, 25)}
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource));
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0);
+ task.start();
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileDataCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0)));
+ }
+
+ @Test
+ public void testMarkFileAndRepairWithInnerSeqSpaceCompactionTask() throws IOException {
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][][] {
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(10, 20), new TimeRange(29, 30), new TimeRange(21, 25)}
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(5, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ InnerSpaceCompactionTask task =
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ Arrays.asList(seqResource1, seqResource2),
+ true,
+ new ReadChunkCompactionPerformer(),
+ 0);
+ Assert.assertFalse(task.start());
+
+ for (TsFileResource resource : tsFileManager.getTsFileList(true)) {
+ Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR);
+ }
+
+ long initialFinishedCompactionTaskNum =
+ CompactionTaskManager.getInstance().getFinishedTaskNum();
+ CompactionScheduleSummary summary = new CompactionScheduleSummary();
+ CompactionScheduler.scheduleCompaction(tsFileManager, 0, summary);
+ Assert.assertEquals(2, summary.getSubmitSeqInnerSpaceCompactionTaskNum());
+
+ int waitSecond = 20;
+ while (CompactionTaskManager.getInstance().getFinishedTaskNum()
+ - initialFinishedCompactionTaskNum
+ < 2) {
+ if (waitSecond == 0) {
+ Assert.fail("Exceed the max time to wait repair compaction");
+ }
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(1));
+ waitSecond--;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(2, tsFileManager.getTsFileList(false).size());
+ for (TsFileResource resource : tsFileManager.getTsFileList(false)) {
+ TsFileResourceUtils.validateTsFileDataCorrectness(resource);
+ }
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0)));
+ }
+
+ @Test
+ public void testMarkFileAndRepairWithInnerUnSeqSpaceCompactionTask() throws IOException {
+ TsFileResource unSeqResource1 = createEmptyFileAndResource(false);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(unSeqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][][] {
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(10, 20), new TimeRange(29, 30), new TimeRange(21, 25)}
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource unSeqResource2 = createEmptyFileAndResource(false);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(unSeqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(5, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ InnerSpaceCompactionTask task =
+ new InnerSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ Arrays.asList(unSeqResource1, unSeqResource2),
+ false,
+ new FastCompactionPerformer(false),
+ 0);
+ Assert.assertFalse(task.start());
+
+ for (TsFileResource resource : tsFileManager.getTsFileList(true)) {
+ Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR);
+ }
+
+ long initialFinishedCompactionTaskNum =
+ CompactionTaskManager.getInstance().getFinishedTaskNum();
+ CompactionScheduleSummary summary = new CompactionScheduleSummary();
+ CompactionScheduler.scheduleCompaction(tsFileManager, 0, summary);
+ Assert.assertEquals(2, summary.getSubmitUnseqInnerSpaceCompactionTaskNum());
+
+ int waitSecond = 20;
+ while (CompactionTaskManager.getInstance().getFinishedTaskNum()
+ - initialFinishedCompactionTaskNum
+ < 2) {
+ if (waitSecond == 0) {
+ Assert.fail("Exceed the max time to wait repair compaction");
+ }
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(1));
+ waitSecond--;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(2, tsFileManager.getTsFileList(false).size());
+ for (TsFileResource resource : tsFileManager.getTsFileList(false)) {
+ TsFileResourceUtils.validateTsFileDataCorrectness(resource);
+ }
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ tsFileManager.getOrCreateSequenceListByTimePartition(0)));
+ }
+
+ @Test
+ public void testMarkFileAndRepairWithCrossSpaceCompactionTask() throws IOException {
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][][] {
+ new TimeRange[][] {
+ new TimeRange[] {new TimeRange(10, 20), new TimeRange(29, 30), new TimeRange(21, 25)}
+ }
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource unSeqResource1 = createEmptyFileAndResource(false);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(unSeqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(5, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ CrossSpaceCompactionTask task =
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ Collections.singletonList(seqResource1),
+ Collections.singletonList(unSeqResource1),
+ new FastCompactionPerformer(true),
+ 0,
+ 0);
+ Assert.assertFalse(task.start());
+
+ for (TsFileResource resource : tsFileManager.getTsFileList(true)) {
+ Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR);
+ }
+
+ long initialFinishedCompactionTaskNum =
+ CompactionTaskManager.getInstance().getFinishedTaskNum();
+ CompactionScheduleSummary summary = new CompactionScheduleSummary();
+ CompactionScheduler.scheduleCompaction(tsFileManager, 0, summary);
+ Assert.assertEquals(1, summary.getSubmitSeqInnerSpaceCompactionTaskNum());
+ Assert.assertEquals(1, summary.getSubmitUnseqInnerSpaceCompactionTaskNum());
+
+ int waitSecond = 20;
+ while (CompactionTaskManager.getInstance().getFinishedTaskNum()
+ - initialFinishedCompactionTaskNum
+ < 2) {
+ if (waitSecond == 0) {
+ Assert.fail("Exceed the max time to wait repair compaction");
+ }
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(1));
+ waitSecond--;
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ Assert.assertEquals(0, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(2, tsFileManager.getTsFileList(false).size());
+ for (TsFileResource resource : tsFileManager.getTsFileList(false)) {
+ TsFileResourceUtils.validateTsFileDataCorrectness(resource);
+ }
+ }
+
+ @Test
+ public void testRepairOverlapBetweenFile() throws IOException {
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(25, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(20, 30), new TimeRange(35, 40)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, false, 0);
+ Assert.assertTrue(task.start());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileDataCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileResourceCorrectness(
+ tsFileManager.getTsFileList(false).get(0)));
+ }
+
+ @Test
+ public void testRepairOverlapBetweenFileWithModFile() throws IOException, IllegalPathException {
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(25, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(20, 30), new TimeRange(35, 40)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ ModificationFile modFile = seqResource2.getModFile();
+ Deletion writedModification =
+ new Deletion(new PartialPath("root.testsg.d1.s1"), Long.MAX_VALUE, 15);
+ modFile.write(writedModification);
+ modFile.close();
+
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, false, 0);
+ Assert.assertTrue(task.start());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0);
+ Assert.assertTrue(TsFileResourceUtils.validateTsFileDataCorrectness(targetResource));
+ Assert.assertTrue(TsFileResourceUtils.validateTsFileResourceCorrectness(targetResource));
+ Assert.assertTrue(targetResource.modFileExists());
+ Assert.assertEquals(1, targetResource.getModFile().getModifications().size());
+ Deletion modification =
+ (Deletion) targetResource.getModFile().getModifications().iterator().next();
+ Assert.assertEquals(writedModification.getFileOffset(), modification.getFileOffset());
+ Assert.assertEquals(writedModification.getEndTime(), modification.getEndTime());
+ }
+
+ @Test
+ public void testScheduleRepairInternalUnsortedFile() throws IOException {
+ DataRegion mockDataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(mockDataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(mockDataRegion.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockDataRegion.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockDataRegion.getTimePartitions()).thenReturn(Collections.singletonList(0L));
+
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(15, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(40, 50), new TimeRange(55, 60)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertTrue(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ long currentTime = System.currentTimeMillis();
+ Path tempDir = Files.createTempDirectory("");
+ tempDir.toFile().deleteOnExit();
+ File logFile =
+ new File(tempDir.toString() + File.separator + currentTime + RepairLogger.repairLogSuffix);
+
+ Files.createFile(logFile.toPath());
+ UnsortedFileRepairTaskScheduler scheduler =
+ new UnsortedFileRepairTaskScheduler(Collections.singletonList(mockDataRegion), logFile);
+ scheduler.run();
+ Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ }
+
+ @Test
+ public void testScheduleRepairOverlapFile() throws IOException {
+ DataRegion mockDataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(mockDataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(mockDataRegion.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockDataRegion.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockDataRegion.getTimePartitions()).thenReturn(Collections.singletonList(0L));
+
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(25, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 50), new TimeRange(55, 60)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ long currentTime = System.currentTimeMillis();
+ Path tempDir = Files.createTempDirectory("");
+ tempDir.toFile().deleteOnExit();
+ File logFile =
+ new File(tempDir.toString() + File.separator + currentTime + RepairLogger.repairLogSuffix);
+
+ Files.createFile(logFile.toPath());
+ UnsortedFileRepairTaskScheduler scheduler =
+ new UnsortedFileRepairTaskScheduler(Collections.singletonList(mockDataRegion), logFile);
+ scheduler.run();
+ Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(1, tsFileManager.getTsFileList(false).size());
+ }
+
+ @Test
+ public void testScheduleRepairOverlapFileAndInternalUnsortedFile() throws IOException {
+ DataRegion mockDataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(mockDataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(mockDataRegion.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockDataRegion.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockDataRegion.getTimePartitions()).thenReturn(Collections.singletonList(0L));
+
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(15, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 50), new TimeRange(55, 60)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ TsFileResource seqResource3 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource3)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(40, 80)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ seqResources.add(seqResource3);
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ long currentTime = System.currentTimeMillis();
+ Path tempDir = Files.createTempDirectory("");
+ tempDir.toFile().deleteOnExit();
+ File logFile =
+ new File(tempDir.toString() + File.separator + currentTime + RepairLogger.repairLogSuffix);
+
+ Files.createFile(logFile.toPath());
+ UnsortedFileRepairTaskScheduler scheduler =
+ new UnsortedFileRepairTaskScheduler(Collections.singletonList(mockDataRegion), logFile);
+ scheduler.run();
+ Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
+ Assert.assertEquals(2, tsFileManager.getTsFileList(false).size());
+ }
+
+ @Test
+ public void testRecoverRepairScheduleSkipRepairedTimePartitionAndMarkFile() throws IOException {
+ DataRegion mockDataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(mockDataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(mockDataRegion.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockDataRegion.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockDataRegion.getTimePartitions()).thenReturn(Collections.singletonList(0L));
+
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(15, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 50), new TimeRange(55, 60)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ TsFileResource seqResource3 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource3)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(40, 80)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResource3.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ seqResources.add(seqResource3);
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ long currentTime = System.currentTimeMillis();
+ Path tempDir = Files.createTempDirectory("");
+ tempDir.toFile().deleteOnExit();
+ File logFile =
+ new File(tempDir.toString() + File.separator + currentTime + RepairLogger.repairLogSuffix);
+ Files.createFile(logFile.toPath());
+ try (RepairLogger logger = new RepairLogger(logFile)) {
+ RepairTimePartition timePartition =
+ new RepairTimePartition(mockDataRegion, 0, System.currentTimeMillis());
+ // record seqResource3 as cannot recover
+ logger.recordRepairedTimePartition(timePartition);
+ }
+ // reset the repair status
+ seqResource3.setTsFileRepairStatus(TsFileRepairStatus.NORMAL);
+
+ UnsortedFileRepairTaskScheduler scheduler =
+ new UnsortedFileRepairTaskScheduler(Collections.singletonList(mockDataRegion), logFile);
+ scheduler.run();
+ Assert.assertEquals(3, tsFileManager.getTsFileList(true).size());
+ // check whether the repair status is marked correctly
+ Assert.assertEquals(TsFileRepairStatus.NEED_TO_REPAIR, seqResource3.getTsFileRepairStatus());
+ Files.deleteIfExists(logFile.toPath());
+ }
+
+ @Test
+ public void testRecoverRepairScheduleSkipRepairedTimePartitionWithDeletedFile()
+ throws IOException {
+ DataRegion mockDataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(mockDataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(mockDataRegion.getDatabaseName()).thenReturn("root.testsg");
+ Mockito.when(mockDataRegion.getDataRegionId()).thenReturn("0");
+ Mockito.when(mockDataRegion.getTimePartitions()).thenReturn(Collections.singletonList(0L));
+
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(15, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 50), new TimeRange(55, 60)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ TsFileResource seqResource3 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(seqResource3)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s1", "s2"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(40, 80)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResource3.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ seqResources.add(seqResource3);
+ tsFileManager.addAll(seqResources, true);
+ Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources));
+
+ long currentTime = System.currentTimeMillis();
+ Path tempDir = Files.createTempDirectory("");
+ tempDir.toFile().deleteOnExit();
+ File logFile =
+ new File(tempDir.toString() + File.separator + currentTime + RepairLogger.repairLogSuffix);
+ Files.createFile(logFile.toPath());
+ try (RepairLogger logger = new RepairLogger(logFile)) {
+ RepairTimePartition timePartition =
+ new RepairTimePartition(mockDataRegion, 0, System.currentTimeMillis());
+ // record seqResource3 as cannot recover
+ logger.recordRepairedTimePartition(timePartition);
+ }
+ // reset the repair status
+ seqResource3.setTsFileRepairStatus(TsFileRepairStatus.NORMAL);
+ // resource3 is deleted
+ tsFileManager.replace(
+ Collections.singletonList(seqResource3),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ 0);
+
+ UnsortedFileRepairTaskScheduler scheduler =
+ new UnsortedFileRepairTaskScheduler(Collections.singletonList(mockDataRegion), logFile);
+ scheduler.run();
+ Assert.assertEquals(2, tsFileManager.getTsFileList(true).size());
+ Files.deleteIfExists(logFile.toPath());
+ }
+
+ @Test
+ public void testTimePartitionFilterFiles() {
+ DataRegion mockDataRegion = Mockito.mock(DataRegion.class);
+ Mockito.when(mockDataRegion.getTsFileManager()).thenReturn(tsFileManager);
+ Mockito.when(mockDataRegion.getTimePartitions()).thenReturn(Collections.singletonList(0L));
+
+ TsFileResource seqResource1 = createEmptyFileAndResourceWithName("100-1-0-0.tsfile", 0, true);
+ TsFileResource seqResource2 = createEmptyFileAndResourceWithName("200-3-0-0.tsfile", 0, true);
+ TsFileResource seqResource3 = createEmptyFileAndResourceWithName("300-5-0-0.tsfile", 0, true);
+ TsFileResource unseqResource1 =
+ createEmptyFileAndResourceWithName("101-2-0-0.tsfile", 0, false);
+ TsFileResource unseqResource2 =
+ createEmptyFileAndResourceWithName("201-4-0-0.tsfile", 0, false);
+ TsFileResource unseqResource3 =
+ createEmptyFileAndResourceWithName("301-6-0-0.tsfile", 0, false);
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ seqResources.add(seqResource3);
+ unseqResources.add(unseqResource1);
+ unseqResources.add(unseqResource2);
+ unseqResources.add(unseqResource3);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ RepairTimePartition timePartition = new RepairTimePartition(mockDataRegion, 0, 250);
+ Assert.assertEquals(4, timePartition.getAllFileSnapshot().size());
+ Assert.assertEquals(2, timePartition.getSeqFileSnapshot().size());
+ Assert.assertEquals(2, timePartition.getUnSeqFileSnapshot().size());
+ }
+
+ @Test
+ public void testEstimateRepairCompactionMemory() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(5, 30)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0);
+ Assert.assertTrue(task.getEstimatedMemoryCost() > 0);
+ task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, false, 0);
+ Assert.assertEquals(0, task.getEstimatedMemoryCost());
+ }
+
+ @Test
+ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException {
+ TsFileResource resource = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(
+ Arrays.asList("s1", "s2", "s3"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4,
+ Arrays.asList(true, false, false));
+ writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(
+ Arrays.asList("s1", "s2", "s3"),
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20)}},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4,
+ Arrays.asList(false, true, true));
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ RepairUnsortedFileCompactionTask task =
+ new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0);
+ Assert.assertTrue(task.start());
+ TsFileResource target = tsFileManager.getTsFileList(false).get(0);
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) {
+ List<AlignedChunkMetadata> chunkMetadataList =
+ reader.getAlignedChunkMetadata("root.testsg.d1");
+ for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) {
+ ChunkMetadata timeChunkMetadata =
+ (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata();
+ Chunk timeChunk = reader.readMemChunk(timeChunkMetadata);
+ List<Chunk> valueChunks = new ArrayList<>();
+ for (IChunkMetadata chunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) {
+ Chunk valueChunk = reader.readMemChunk((ChunkMetadata) chunkMetadata);
+ valueChunks.add(valueChunk);
+ }
+ AlignedChunkReader chunkReader = new AlignedChunkReader(timeChunk, valueChunks, null);
+ while (chunkReader.hasNextSatisfiedPage()) {
+ BatchData batchData = chunkReader.nextPageData();
+ IPointReader pointReader = batchData.getBatchDataIterator();
+ while (pointReader.hasNextTimeValuePair()) {
+ TimeValuePair timeValuePair = pointReader.nextTimeValuePair();
+ for (Object value : timeValuePair.getValues()) {
+ if (value == null) {
+ Assert.fail();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java
index 4373ec85bde..102a0d35cb0 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/MultiTsFileDeviceIteratorTest.java
@@ -457,7 +457,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG);
tsFileManager.replace(
- tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0);
tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL);
validateSeqFiles(true);
@@ -603,7 +603,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG);
tsFileManager.replace(
- tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0);
tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL);
validateSeqFiles(true);
@@ -747,7 +747,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG);
tsFileManager.replace(
- tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0);
tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL);
validateSeqFiles(true);
@@ -899,7 +899,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG);
tsFileManager.replace(
- tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0);
tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL);
validateSeqFiles(true);
@@ -1043,7 +1043,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG);
tsFileManager.replace(
- tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0);
tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL);
validateSeqFiles(true);
@@ -1195,7 +1195,7 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG);
tsFileManager.replace(
- tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0, true);
+ tsFileManager.getTsFileList(true), Collections.emptyList(), targetResources, 0);
tsFileManager.getTsFileList(true).get(0).setStatusForTest(TsFileResourceStatus.NORMAL);
validateSeqFiles(true);
diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index 8f560037166..d0320a54081 100644
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@ -1203,6 +1203,9 @@ service IConfigNodeRPCService {
/** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */
common.TSStatus clearCache()
+ /** Check and repair unsorted tsfile by compaction */
+ common.TSStatus repairData()
+
/** Load configuration on all DataNodes */
common.TSStatus loadConfiguration()
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
index 90b24f68c6d..5af2ef4657d 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
@@ -668,6 +668,8 @@ service IDataNodeRPCService {
common.TSStatus settle(common.TSettleReq req)
+ common.TSStatus repairData()
+
common.TSStatus clearCache()
common.TSStatus loadConfiguration()