You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/08/13 06:41:25 UTC
[iotdb] branch master updated: [IOTDB-4117] fix compact bug ArithmeticException: / by zero (#6967)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e9d4b0138f [IOTDB-4117] fix compact bug ArithmeticException: / by zero (#6967)
e9d4b0138f is described below
commit e9d4b0138f6fd241c15dffd35e7ebdc540a81582
Author: Jamber <ja...@sina.com>
AuthorDate: Sat Aug 13 14:41:20 2022 +0800
[IOTDB-4117] fix compact bug ArithmeticException: / by zero (#6967)
---
.../compaction/cross/CrossSpaceCompactionTask.java | 3 +-
.../utils/ReadPointCrossCompactionEstimator.java | 44 +++++++++++++++++-----
.../impl/ReadPointCompactionPerformer.java | 2 +-
3 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
index ab6725440e..2ffe154f23 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
@@ -173,7 +173,8 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
LOGGER.error(
"{}-{} [Compaction] Meet errors in cross space compaction.",
storageGroupName,
- dataRegionId);
+ dataRegionId,
+ throwable);
} else {
LOGGER.warn("{}-{} [Compaction] Compaction interrupted", storageGroupName, dataRegionId);
// clean the interrupted flag
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/utils/ReadPointCrossCompactionEstimator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/utils/ReadPointCrossCompactionEstimator.java
index e025dd1a3e..4202035527 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/utils/ReadPointCrossCompactionEstimator.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/utils/ReadPointCrossCompactionEstimator.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.engine.compaction.cross.utils;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceEstimator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -25,12 +26,18 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class ReadPointCrossCompactionEstimator extends AbstractCrossSpaceEstimator {
+ private static final Logger logger =
+ LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME);
+
// the max cost of reading source seq file among all source seq files of this cross compaction
// task
private long maxCostOfReadingSeqFile;
@@ -77,6 +84,12 @@ public class ReadPointCrossCompactionEstimator extends AbstractCrossSpaceEstimat
// it means the max size of a timeseries in this file when reading all of its chunk into memory.
// Not only reading chunk into chunk cache, but also need to deserialize data point into merge
// reader, so we have to double the cost here.
+ if (fileInfo[0] == 0) { // If totalChunkNum ==0, i.e. this unSeq tsFile has no chunk.
+ logger.warn(
+ "calculateReadingUnseqFile(), find 1 empty unSeq tsFile: {}.",
+ unseqResource.getTsFilePath());
+ return 0;
+ }
return 2 * concurrentSeriesNum * (unseqResource.getTsFileSize() * fileInfo[1] / fileInfo[0]);
}
@@ -93,8 +106,17 @@ public class ReadPointCrossCompactionEstimator extends AbstractCrossSpaceEstimat
// it is max aligned series num of one device when tsfile contains aligned series,
// else is sub compaction task num.
int concurrentSeriesNum = fileInfo[2] == -1 ? subCompactionTaskNum : fileInfo[2];
- long seqFileCost =
- concurrentSeriesNum * (seqResource.getTsFileSize() * fileInfo[1] / fileInfo[0]);
+ long seqFileCost = 0;
+ if (fileInfo[0] == 0) { // If totalChunkNum ==0, i.e. this seq tsFile has no chunk.
+ logger.warn(
+ "calculateReadingSeqFiles(), find 1 empty seq tsFile: {}.",
+ seqResource.getTsFilePath());
+ seqFileCost = 0;
+ } else {
+ seqFileCost =
+ concurrentSeriesNum * (seqResource.getTsFileSize() * fileInfo[1] / fileInfo[0]);
+ }
+
if (seqFileCost > maxCostOfReadingSeqFile) {
// Only one seq file will be read at the same time.
// not only reading chunk into chunk cache, but also need to deserialize data point into
@@ -121,16 +143,18 @@ public class ReadPointCrossCompactionEstimator extends AbstractCrossSpaceEstimat
// add seq file metadata size
cost += reader.getFileMetadataSize();
// add max chunk group size of this seq tsfile
- cost +=
- seqResource.getTsFileSize()
- * maxSeqChunkNumInDeviceList.get(0).left
- / maxSeqChunkNumInDeviceList.get(0).right;
+ int totalSeqChunkNum = maxSeqChunkNumInDeviceList.get(0).right;
+ if (totalSeqChunkNum > 0) {
+ cost +=
+ seqResource.getTsFileSize() * maxSeqChunkNumInDeviceList.get(0).left / totalSeqChunkNum;
+ }
}
// add max chunk group size of overlapped unseq tsfile
- cost +=
- unseqResource.getTsFileSize()
- * maxUnseqChunkNumInDevice.left
- / maxUnseqChunkNumInDevice.right;
+ int totalUnSeqChunkNum = maxUnseqChunkNumInDevice.right;
+ if (totalUnSeqChunkNum > 0) {
+ cost += unseqResource.getTsFileSize() * maxUnseqChunkNumInDevice.left / totalUnSeqChunkNum;
+ }
+
return cost;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
index 81e29aa7c6..0b8df320ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java
@@ -300,7 +300,7 @@ public class ReadPointCompactionPerformer
TsFileResource resource, TsFileSequenceReader reader, String device, String measurement)
throws IllegalPathException, IOException {
List<ChunkMetadata> chunkMetadata =
- reader.getChunkMetadataList(new PartialPath(device, measurement));
+ reader.getChunkMetadataList(new PartialPath(device, measurement), true);
if (chunkMetadata.size() > 0) {
chunkMetadata.get(0).setFilePath(resource.getTsFilePath());
Chunk chunk = ChunkCache.getInstance().get(chunkMetadata.get(0));