You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2021/06/05 10:42:38 UTC
[iotdb] 01/02: [DEBUG] Compaction error schema add log (#3324)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch 0.12-debug-compaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1901c0a6237c3595a03de7c24dc88d2f0ed4c3fc
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Thu Jun 3 21:24:24 2021 +0800
[DEBUG] Compaction error schema add log (#3324)
Co-authored-by: zhanglingzhe <su...@foxmail.com>
---
.../level/LevelCompactionTsFileManagement.java | 4 ++--
.../db/engine/compaction/utils/CompactionUtils.java | 21 +++++++++------------
.../java/org/apache/iotdb/db/metadata/MManager.java | 1 +
.../db/engine/compaction/CompactionChunkTest.java | 2 +-
.../compaction/LevelCompactionRecoverTest.java | 12 +++++-------
5 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index dc37e8e..cbc8b24 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -29,7 +29,7 @@ import org.apache.iotdb.db.engine.compaction.utils.CompactionUtils;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -526,7 +526,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
}
}
}
- } catch (IOException | IllegalPathException | InterruptedException e) {
+ } catch (IOException | InterruptedException | MetadataException e) {
logger.error("recover level tsfile management error ", e);
} finally {
if (logFile.exists()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
index a6aa68c..e056fe0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.service.IoTDB;
@@ -178,7 +177,7 @@ public class CompactionUtils {
RestorableTsFileIOWriter writer,
Map<String, List<Modification>> modificationCache,
List<Modification> modifications)
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
Map<Long, TimeValuePair> timeValuePairMap = new TreeMap<>();
Map<TsFileSequenceReader, List<ChunkMetadata>> readerChunkMetadataMap = entry.getValue();
readByDeserializePageMerge(
@@ -198,15 +197,10 @@ public class CompactionUtils {
return;
}
IChunkWriter chunkWriter;
- try {
- chunkWriter =
- new ChunkWriterImpl(
- IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true);
- } catch (MetadataException e) {
- // this may caused in IT by restart
- logger.error("{} get schema {} error, skip this sensor", device, entry.getKey(), e);
- return;
- }
+ chunkWriter =
+ new ChunkWriterImpl(
+ IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true);
+
for (TimeValuePair timeValuePair : timeValuePairMap.values()) {
writeTVPair(timeValuePair, chunkWriter);
targetResource.updateStartTime(device, timeValuePair.getTimestamp());
@@ -259,7 +253,7 @@ public class CompactionUtils {
Set<String> devices,
boolean sequence,
List<Modification> modifications)
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(targetResource.getTsFile());
Map<String, TsFileSequenceReader> tsFileSequenceReaderMap = new HashMap<>();
Map<String, List<Modification>> modificationCache = new HashMap<>();
@@ -304,6 +298,9 @@ public class CompactionUtils {
if (sensorChunkMetadataListMap.size() <= 0) {
if (chunkMetadataListIteratorCache.get(reader).hasNext()) {
sensorChunkMetadataListMap = chunkMetadataListIteratorCache.get(reader).next();
+ for (String sensor : sensorChunkMetadataListMap.keySet()) {
+ logger.error("chunkMetadataListIterator get measurement:{}", device + "." + sensor);
+ }
chunkMetadataListCacheForMerge.put(reader, sensorChunkMetadataListMap);
} else {
continue;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8224995..b74ea34 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -998,6 +998,7 @@ public class MManager {
if (leaf != null) {
return ((MeasurementMNode) leaf).getSchema();
}
+ logger.error("cannot get schema {}", device.getDevice() + "." + measurement);
return null;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
index 5e7276a..bf958c4 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
@@ -152,7 +152,7 @@ public class CompactionChunkTest extends LevelCompactionTest {
}
@Test
- public void testDeserializeMerge() throws IOException, IllegalPathException {
+ public void testDeserializeMerge() throws IOException, MetadataException {
Map<String, Map<TsFileSequenceReader, List<ChunkMetadata>>> measurementChunkMetadataMap =
new HashMap<>();
List<TsFileResource> sourceTsfileResources = seqResources.subList(0, 2);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
index 7adffc2..473777e 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
@@ -80,7 +80,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished */
@Test
- public void testCompactionMergeRecoverMergeFinished() throws IOException, IllegalPathException {
+ public void testCompactionMergeRecoverMergeFinished() throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -171,7 +171,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished, delete one offset */
@Test
public void testCompactionMergeRecoverMergeFinishedAndDeleteOneOffset()
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -286,7 +286,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished, delete one device - offset */
@Test
public void testCompactionMergeRecoverMergeFinishedAndDeleteOneDeviceWithOffset()
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -406,8 +406,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished,unseq */
@Test
- public void testCompactionMergeRecoverMergeFinishedUnseq()
- throws IOException, IllegalPathException {
+ public void testCompactionMergeRecoverMergeFinishedUnseq() throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -638,8 +637,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished but no finish log */
@Test
- public void testCompactionMergeRecoverMergeFinishedNoLog()
- throws IOException, IllegalPathException {
+ public void testCompactionMergeRecoverMergeFinishedNoLog() throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);