You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2021/06/03 14:40:13 UTC
[iotdb] branch rel/0.12-debug-compaction updated: add logs
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch rel/0.12-debug-compaction
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12-debug-compaction by this push:
new 13b85ca add logs
13b85ca is described below
commit 13b85caa56b76a8e248d1e0a24f13e66bb366379
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu Jun 3 22:35:13 2021 +0800
add logs
---
.../main/java/org/apache/iotdb/SessionExample.java | 50 +++++++++++-----------
.../engine/compaction/utils/CompactionUtils.java | 9 ++--
.../org/apache/iotdb/db/metadata/MManager.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 4 ++
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 49 ++++++++++++++++++++-
5 files changed, 84 insertions(+), 30 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index 386b4e4..b91d49f 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -66,30 +66,30 @@ public class SessionExample {
}
createTimeseries();
- createMultiTimeseries();
- insertRecord();
+ // createMultiTimeseries();
+ // insertRecord();
insertTablet();
- insertTablets();
- insertRecords();
- nonQuery();
- query();
- queryWithTimeout();
- rawDataQuery();
- queryByIterator();
- deleteData();
- deleteTimeseries();
- setTimeout();
-
- sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root");
- sessionEnableRedirect.setEnableQueryRedirection(true);
- sessionEnableRedirect.open(false);
-
- // set session fetchSize
- sessionEnableRedirect.setFetchSize(10000);
-
- insertRecord4Redirect();
- query4Redirect();
- sessionEnableRedirect.close();
+ // insertTablets();
+ // insertRecords();
+ // nonQuery();
+ // query();
+ // queryWithTimeout();
+ // rawDataQuery();
+ // queryByIterator();
+ // deleteData();
+ // deleteTimeseries();
+ // setTimeout();
+ //
+ // sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root");
+ // sessionEnableRedirect.setEnableQueryRedirection(true);
+ // sessionEnableRedirect.open(false);
+ //
+ // // set session fetchSize
+ // sessionEnableRedirect.setFetchSize(10000);
+ //
+ // insertRecord4Redirect();
+ // query4Redirect();
+ // sessionEnableRedirect.close();
session.close();
}
@@ -331,7 +331,7 @@ public class SessionExample {
// Method 1 to add tablet data
long timestamp = System.currentTimeMillis();
- for (long row = 0; row < 100; row++) {
+ for (long row = 0; row < 10000000; row++) {
int rowIndex = tablet.rowSize++;
tablet.addTimestamp(rowIndex, timestamp);
for (int s = 0; s < 3; s++) {
@@ -354,7 +354,7 @@ public class SessionExample {
long[] timestamps = tablet.timestamps;
Object[] values = tablet.values;
- for (long time = 0; time < 100; time++) {
+ for (long time = 0; time < 1000; time++) {
int row = tablet.rowSize++;
timestamps[row] = time;
for (int i = 0; i < 3; i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
index e056fe0..9b5c7e9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
@@ -196,8 +196,10 @@ public class CompactionUtils {
if (isChunkMetadataEmpty) {
return;
}
- IChunkWriter chunkWriter;
- chunkWriter =
+ logger.warn(
+ "++++++ new ChunkWriterImpl in writeByDeserializePageMerge: "
+ + new PartialPath(device, entry.getKey()).getFullPath());
+ IChunkWriter chunkWriter =
new ChunkWriterImpl(
IoTDB.metaManager.getSeriesSchema(new PartialPath(device), entry.getKey()), true);
@@ -299,7 +301,8 @@ public class CompactionUtils {
if (chunkMetadataListIteratorCache.get(reader).hasNext()) {
sensorChunkMetadataListMap = chunkMetadataListIteratorCache.get(reader).next();
for (String sensor : sensorChunkMetadataListMap.keySet()) {
- logger.error("chunkMetadataListIterator get measurement:{}", device + "." + sensor);
+ logger.warn(
+ "+++++ chunkMetadataListIterator get measurement:{}.{}", device, sensor);
}
chunkMetadataListCacheForMerge.put(reader, sensorChunkMetadataListMap);
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index b74ea34..ce44d49 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -998,7 +998,7 @@ public class MManager {
if (leaf != null) {
return ((MeasurementMNode) leaf).getSchema();
}
- logger.error("cannot get schema {}", device.getDevice() + "." + measurement);
+ logger.warn("+++++ cannot get schema {}", device.getDevice() + "." + measurement);
return null;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 211a688..17ffdc9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1323,6 +1323,10 @@ public class TsFileSequenceReader implements AutoCloseable {
measurementChunkMetadataList
.computeIfAbsent(timeseriesMetadata.getMeasurementId(), m -> new ArrayList<>())
.addAll(timeseriesMetadata.getChunkMetadataList());
+ logger.warn(
+ "----- getMeasurementChunkMetadataListMapIterator#next(): {}.{}",
+ device,
+ timeseriesMetadata.getMeasurementId());
}
return measurementChunkMetadataList;
} catch (IOException e) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 3df2eb9..a7373ab 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.BytesUtils;
@@ -47,11 +48,15 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.TreeMap;
/**
@@ -232,14 +237,19 @@ public class TsFileIOWriter {
// serialize the SEPARATOR of MetaData
ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
+ logger.warn("-------------START: group ChunkMetadata by series-----------------");
+ Set<Path> expectedPaths = new HashSet<>();
// group ChunkMetadata by series
Map<Path, List<ChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
+ expectedPaths.add(series);
+ logger.warn("++++++ endFile: {}", series.getFullPath());
chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
}
}
+ logger.warn("-------------END: group ChunkMetadata by series-------------------");
MetadataIndexNode metadataIndex = flushMetadataIndex(chunkMetadataListMap);
TsFileMetadata tsFileMetaData = new TsFileMetadata();
@@ -274,7 +284,44 @@ public class TsFileIOWriter {
if (resourceLogger.isDebugEnabled() && file != null) {
resourceLogger.debug("{} writer is closed.", file.getName());
}
- canWrite = false;
+
+ logger.warn("-------------START: write check-----------------");
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) {
+ Set<Path> actualPaths1 = new HashSet<>();
+ Set<Path> actualPaths2 = new HashSet<>();
+
+ for (Entry<String, List<TimeseriesMetadata>> entry :
+ reader.getAllTimeseriesMetadata().entrySet()) {
+ for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ actualPaths1.add(new Path(entry.getKey(), timeseriesMetadata.getMeasurementId()));
+ }
+
+ Iterator<Map<String, List<ChunkMetadata>>> iterator =
+ reader.getMeasurementChunkMetadataListMapIterator(entry.getKey());
+ while (iterator.hasNext()) {
+ for (String m : iterator.next().keySet()) {
+ actualPaths2.add(new Path(entry.getKey(), m));
+ }
+ }
+ }
+
+ if (!actualPaths1.equals(expectedPaths)) {
+ logger.error("!!!!!!!!!!!! - 1 reader.getAllTimeseriesMetadata()");
+ logger.error(Arrays.toString(expectedPaths.toArray()));
+ logger.error(Arrays.toString(actualPaths1.toArray()));
+ logger.error("!!!!!!!!!!!!");
+ }
+
+ if (!actualPaths2.equals(expectedPaths)) {
+ logger.error("!!!!!!!!!!!! - 2 reader.getMeasurementChunkMetadataListMapIterator");
+ logger.error(Arrays.toString(expectedPaths.toArray()));
+ logger.error(Arrays.toString(actualPaths2.toArray()));
+ logger.error("!!!!!!!!!!!!");
+ }
+ } finally {
+ logger.warn("-------------END: write check-----------------");
+ canWrite = false;
+ }
}
/**