You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2021/06/05 10:42:37 UTC
[iotdb] branch 0.12-debug-compaction created (now 13b85ca)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch 0.12-debug-compaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 13b85ca add logs
This branch includes the following new commits:
new 1901c0a [DEBUG] Compaction error schema add log (#3324)
new 13b85ca add logs
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: add logs
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch 0.12-debug-compaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 13b85caa56b76a8e248d1e0a24f13e66bb366379
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu Jun 3 22:35:13 2021 +0800
add logs
---
.../main/java/org/apache/iotdb/SessionExample.java | 50 +++++++++++-----------
.../engine/compaction/utils/CompactionUtils.java | 9 ++--
.../org/apache/iotdb/db/metadata/MManager.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 4 ++
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 49 ++++++++++++++++++++-
5 files changed, 84 insertions(+), 30 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index 386b4e4..b91d49f 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -66,30 +66,30 @@ public class SessionExample {
}
createTimeseries();
- createMultiTimeseries();
- insertRecord();
+ // createMultiTimeseries();
+ // insertRecord();
insertTablet();
- insertTablets();
- insertRecords();
- nonQuery();
- query();
- queryWithTimeout();
- rawDataQuery();
- queryByIterator();
- deleteData();
- deleteTimeseries();
- setTimeout();
-
- sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root");
- sessionEnableRedirect.setEnableQueryRedirection(true);
- sessionEnableRedirect.open(false);
-
- // set session fetchSize
- sessionEnableRedirect.setFetchSize(10000);
-
- insertRecord4Redirect();
- query4Redirect();
- sessionEnableRedirect.close();
+ // insertTablets();
+ // insertRecords();
+ // nonQuery();
+ // query();
+ // queryWithTimeout();
+ // rawDataQuery();
+ // queryByIterator();
+ // deleteData();
+ // deleteTimeseries();
+ // setTimeout();
+ //
+ // sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root");
+ // sessionEnableRedirect.setEnableQueryRedirection(true);
+ // sessionEnableRedirect.open(false);
+ //
+ // // set session fetchSize
+ // sessionEnableRedirect.setFetchSize(10000);
+ //
+ // insertRecord4Redirect();
+ // query4Redirect();
+ // sessionEnableRedirect.close();
session.close();
}
@@ -331,7 +331,7 @@ public class SessionExample {
// Method 1 to add tablet data
long timestamp = System.currentTimeMillis();
- for (long row = 0; row < 100; row++) {
+ for (long row = 0; row < 10000000; row++) {
int rowIndex = tablet.rowSize++;
tablet.addTimestamp(rowIndex, timestamp);
for (int s = 0; s < 3; s++) {
@@ -354,7 +354,7 @@ public class SessionExample {
long[] timestamps = tablet.timestamps;
Object[] values = tablet.values;
- for (long time = 0; time < 100; time++) {
+ for (long time = 0; time < 1000; time++) {
int row = tablet.rowSize++;
timestamps[row] = time;
for (int i = 0; i < 3; i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
index e056fe0..9b5c7e9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
@@ -196,8 +196,10 @@ public class CompactionUtils {
if (isChunkMetadataEmpty) {
return;
}
- IChunkWriter chunkWriter;
- chunkWriter =
+ logger.warn(
+ "++++++ new ChunkWriterImpl in writeByDeserializePageMerge: "
+ + new PartialPath(device, entry.getKey()).getFullPath());
+ IChunkWriter chunkWriter =
new ChunkWriterImpl(
IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true);
@@ -299,7 +301,8 @@ public class CompactionUtils {
if (chunkMetadataListIteratorCache.get(reader).hasNext()) {
sensorChunkMetadataListMap = chunkMetadataListIteratorCache.get(reader).next();
for (String sensor : sensorChunkMetadataListMap.keySet()) {
- logger.error("chunkMetadataListIterator get measurement:{}", device + "." + sensor);
+ logger.warn(
+ "+++++ chunkMetadataListIterator get measurement:{}.{}", device, sensor);
}
chunkMetadataListCacheForMerge.put(reader, sensorChunkMetadataListMap);
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index b74ea34..ce44d49 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -998,7 +998,7 @@ public class MManager {
if (leaf != null) {
return ((MeasurementMNode) leaf).getSchema();
}
- logger.error("cannot get schema {}", device.getDevice() + "." + measurement);
+ logger.warn("+++++ cannot get schema {}", device.getDevice() + "." + measurement);
return null;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 211a688..17ffdc9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1323,6 +1323,10 @@ public class TsFileSequenceReader implements AutoCloseable {
measurementChunkMetadataList
.computeIfAbsent(timeseriesMetadata.getMeasurementId(), m -> new ArrayList<>())
.addAll(timeseriesMetadata.getChunkMetadataList());
+ logger.warn(
+ "----- getMeasurementChunkMetadataListMapIterator#next(): {}.{}",
+ device,
+ timeseriesMetadata.getMeasurementId());
}
return measurementChunkMetadataList;
} catch (IOException e) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 3df2eb9..a7373ab 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BytesUtils;
@@ -47,11 +48,15 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.TreeMap;
/**
@@ -232,14 +237,19 @@ public class TsFileIOWriter {
// serialize the SEPARATOR of MetaData
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
+ logger.warn("-------------START: group ChunkMetadata by series-----------------");
+ Set<Path> expectedPaths = new HashSet<>();
// group ChunkMetadata by series
Map<Path, List<ChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
+ expectedPaths.add(series);
+ logger.warn("++++++ endFile: {}", series.getFullPath());
chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
}
}
+ logger.warn("-------------END: group ChunkMetadata by series-------------------");
MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap);
TsFileMetadata tsFileMetaData = new TsFileMetadata();
@@ -274,7 +284,44 @@ public class TsFileIOWriter {
if (resourceLogger.isDebugEnabled() && file != null) {
resourceLogger.debug("{} writer is closed.", file.getName());
}
- canWrite = false;
+
+ logger.warn("-------------START: write check-----------------");
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) {
+ Set<Path> actualPaths1 = new HashSet<>();
+ Set<Path> actualPaths2 = new HashSet<>();
+
+ for (Entry<String, List<TimeseriesMetadata>> entry :
+ reader.getAllTimeseriesMetadata().entrySet()) {
+ for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ actualPaths1.add(new Path(entry.getKey(), timeseriesMetadata.getMeasurementId()));
+ }
+
+ Iterator<Map<String, List<ChunkMetadata>>> iterator =
+ reader.getMeasurementChunkMetadataListMapIterator(entry.getKey());
+ while (iterator.hasNext()) {
+ for (String m : iterator.next().keySet()) {
+ actualPaths2.add(new Path(entry.getKey(), m));
+ }
+ }
+ }
+
+ if (!actualPaths1.equals(expectedPaths)) {
+ logger.error("!!!!!!!!!!!! - 1 reader.getAllTimeseriesMetadata()");
+ logger.error(Arrays.toString(expectedPaths.toArray()));
+ logger.error(Arrays.toString(actualPaths1.toArray()));
+ logger.error("!!!!!!!!!!!!");
+ }
+
+ if (!actualPaths2.equals(expectedPaths)) {
+ logger.error("!!!!!!!!!!!! - 2 reader.getMeasurementChunkMetadataListMapIterator");
+ logger.error(Arrays.toString(expectedPaths.toArray()));
+ logger.error(Arrays.toString(actualPaths2.toArray()));
+ logger.error("!!!!!!!!!!!!");
+ }
+ } finally {
+ logger.warn("-------------END: write check-----------------");
+ canWrite = false;
+ }
}
/**
[iotdb] 01/02: [DEBUG] Compaction error schema add log (#3324)
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch 0.12-debug-compaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1901c0a6237c3595a03de7c24dc88d2f0ed4c3fc
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Thu Jun 3 21:24:24 2021 +0800
[DEBUG] Compaction error schema add log (#3324)
Co-authored-by: zhanglingzhe <su...@foxmail.com>
---
.../level/LevelCompactionTsFileManagement.java | 4 ++--
.../db/engine/compaction/utils/CompactionUtils.java | 21 +++++++++------------
.../java/org/apache/iotdb/db/metadata/MManager.java | 1 +
.../db/engine/compaction/CompactionChunkTest.java | 2 +-
.../compaction/LevelCompactionRecoverTest.java | 12 +++++-------
5 files changed, 18 insertions(+), 22 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index dc37e8e..cbc8b24 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -29,7 +29,7 @@ import org.apache.iotdb.db.engine.compaction.utils.CompactionUtils;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -526,7 +526,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
}
}
}
- } catch (IOException | IllegalPathException | InterruptedException e) {
+ } catch (IOException | InterruptedException | MetadataException e) {
logger.error("recover level tsfile management error ", e);
} finally {
if (logFile.exists()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
index a6aa68c..e056fe0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.service.IoTDB;
@@ -178,7 +177,7 @@ public class CompactionUtils {
RestorableTsFileIOWriter writer,
Map<String, List<Modification>> modificationCache,
List<Modification> modifications)
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
Map<Long, TimeValuePair> timeValuePairMap = new TreeMap<>();
Map<TsFileSequenceReader, List<ChunkMetadata>> readerChunkMetadataMap = entry.getValue();
readByDeserializePageMerge(
@@ -198,15 +197,10 @@ public class CompactionUtils {
return;
}
IChunkWriter chunkWriter;
- try {
- chunkWriter =
- new ChunkWriterImpl(
- IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true);
- } catch (MetadataException e) {
- // this may caused in IT by restart
- logger.error("{} get schema {} error, skip this sensor", device, entry.getKey(), e);
- return;
- }
+ chunkWriter =
+ new ChunkWriterImpl(
+ IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true);
+
for (TimeValuePair timeValuePair : timeValuePairMap.values()) {
writeTVPair(timeValuePair, chunkWriter);
targetResource.updateStartTime(device, timeValuePair.getTimestamp());
@@ -259,7 +253,7 @@ public class CompactionUtils {
Set<String> devices,
boolean sequence,
List<Modification> modifications)
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(targetResource.getTsFile());
Map<String, TsFileSequenceReader> tsFileSequenceReaderMap = new HashMap<>();
Map<String, List<Modification>> modificationCache = new HashMap<>();
@@ -304,6 +298,9 @@ public class CompactionUtils {
if (sensorChunkMetadataListMap.size() <= 0) {
if (chunkMetadataListIteratorCache.get(reader).hasNext()) {
sensorChunkMetadataListMap = chunkMetadataListIteratorCache.get(reader).next();
+ for (String sensor : sensorChunkMetadataListMap.keySet()) {
+ logger.error("chunkMetadataListIterator get measurement:{}", device + "." + sensor);
+ }
chunkMetadataListCacheForMerge.put(reader, sensorChunkMetadataListMap);
} else {
continue;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8224995..b74ea34 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -998,6 +998,7 @@ public class MManager {
if (leaf != null) {
return ((MeasurementMNode) leaf).getSchema();
}
+ logger.error("cannot get schema {}", device.getDevice() + "." + measurement);
return null;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
index 5e7276a..bf958c4 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionChunkTest.java
@@ -152,7 +152,7 @@ public class CompactionChunkTest extends LevelCompactionTest {
}
@Test
- public void testDeserializeMerge() throws IOException, IllegalPathException {
+ public void testDeserializeMerge() throws IOException, MetadataException {
Map<String, Map<TsFileSequenceReader, List<ChunkMetadata>>> measurementChunkMetadataMap =
new HashMap<>();
List<TsFileResource> sourceTsfileResources = seqResources.subList(0, 2);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
index 7adffc2..473777e 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/LevelCompactionRecoverTest.java
@@ -80,7 +80,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished */
@Test
- public void testCompactionMergeRecoverMergeFinished() throws IOException, IllegalPathException {
+ public void testCompactionMergeRecoverMergeFinished() throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -171,7 +171,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished, delete one offset */
@Test
public void testCompactionMergeRecoverMergeFinishedAndDeleteOneOffset()
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -286,7 +286,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished, delete one device - offset */
@Test
public void testCompactionMergeRecoverMergeFinishedAndDeleteOneDeviceWithOffset()
- throws IOException, IllegalPathException {
+ throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -406,8 +406,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished,unseq */
@Test
- public void testCompactionMergeRecoverMergeFinishedUnseq()
- throws IOException, IllegalPathException {
+ public void testCompactionMergeRecoverMergeFinishedUnseq() throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);
@@ -638,8 +637,7 @@ public class LevelCompactionRecoverTest extends LevelCompactionTest {
/** compaction recover merge finished but no finish log */
@Test
- public void testCompactionMergeRecoverMergeFinishedNoLog()
- throws IOException, IllegalPathException {
+ public void testCompactionMergeRecoverMergeFinishedNoLog() throws IOException, MetadataException {
LevelCompactionTsFileManagement levelCompactionTsFileManagement =
new LevelCompactionTsFileManagement(COMPACTION_TEST_SG, tempSGDir.getPath());
levelCompactionTsFileManagement.addAll(seqResources, true);