You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2023/05/18 10:45:33 UTC
[iotdb] 01/01: [To rel/1.1][IOTDB-5897] Fix NullPointerException in compaction (#9886)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch fix-compaction-null-pointer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 921e7defd517a1d7e1a70657ec4f08860df2b2f8
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Thu May 18 18:35:10 2023 +0800
[To rel/1.1][IOTDB-5897] Fix NullPointerException in compaction (#9886)
---
.../execute/utils/MultiTsFileDeviceIterator.java | 3 +-
.../ReadChunkCompactionPerformerAlignedTest.java | 76 ++++++++++++++++++++++
.../compaction/utils/CompactionCheckerUtils.java | 4 ++
3 files changed, 82 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 967338d1f8..6e66ac5331 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -359,7 +359,8 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
if (modification.getDevice().equals(currentDevice.left)) {
for (int i = 0; i < valueChunkMetadataList.size(); ++i) {
IChunkMetadata chunkMetadata = valueChunkMetadataList.get(i);
- if (modification.getMeasurement().equals(chunkMetadata.getMeasurementUid())) {
+ if (chunkMetadata != null
+ && modification.getMeasurement().equals(chunkMetadata.getMeasurementUid())) {
modificationForCurDevice.get(i).add(modification);
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
index 09997f8d7b..66cba77a90 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
@@ -32,14 +32,21 @@ import org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils;
import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer;
import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils;
+import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
+import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
import org.apache.commons.io.FileUtils;
import org.junit.After;
@@ -652,4 +659,73 @@ public class ReadChunkCompactionPerformerAlignedTest {
new ArrayList<>());
CompactionCheckerUtils.validDataByValueList(originData, compactedData);
}
+
+ @Test
+ public void testEmptyChunkWithModification() throws Exception {
+ List<IMeasurementSchema> schemas = new ArrayList<>();
+ schemas.add(new MeasurementSchema("s0", TSDataType.DOUBLE));
+ schemas.add(new MeasurementSchema("s1", TSDataType.FLOAT));
+ schemas.add(new MeasurementSchema("s2", TSDataType.INT64));
+ schemas.add(new MeasurementSchema("s3", TSDataType.INT32));
+ schemas.add(new MeasurementSchema("s4", TSDataType.TEXT));
+ schemas.add(new MeasurementSchema("s5", TSDataType.BOOLEAN));
+ Map<PartialPath, List<TimeValuePair>> originData = new HashMap<>();
+ List<TsFileResource> resources = new ArrayList<>();
+ for (int i = 1; i <= 5; i++) {
+ TsFileIOWriter writer =
+ new TsFileIOWriter(new File(dataDirectory, String.format("%d-%d-0-0.tsfile", i, i)));
+ AlignedChunkWriterImpl alignedChunkWriter = new AlignedChunkWriterImpl(schemas);
+ for (int j = i * 100; j < i * 100 + 100; j++) {
+ TsPrimitiveType[] values = {
+ new TsPrimitiveType.TsDouble(0.0D),
+ new TsPrimitiveType.TsFloat(0.0F),
+ null,
+ null,
+ new TsPrimitiveType.TsBinary(new Binary("")),
+ new TsPrimitiveType.TsBoolean(false)
+ };
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s0"), k -> new ArrayList<>())
+ .add(new TimeValuePair(j, values[0]));
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s1"), k -> new ArrayList<>())
+ .add(new TimeValuePair(j, values[1]));
+ originData.computeIfAbsent(new PartialPath("root.sg.d1.s2"), k -> null);
+ originData.computeIfAbsent(new PartialPath("root.sg.d1.s3"), k -> null);
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s4"), k -> new ArrayList<>())
+ .add(new TimeValuePair(j, values[4]));
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s5"), k -> new ArrayList<>())
+ .add(new TimeValuePair(j, values[5]));
+ alignedChunkWriter.write(j, values);
+ }
+ writer.startChunkGroup("root.sg.d1");
+ alignedChunkWriter.writeToFileWriter(writer);
+ writer.endChunkGroup();
+ writer.endFile();
+ TsFileResource resource = new TsFileResource(writer.getFile(), TsFileResourceStatus.NORMAL);
+ resource
+ .getModFile()
+ .write(new Deletion(new PartialPath("root.sg.d1.*"), i * 100, i * 100 + 20));
+ resource.getModFile().close();
+ int finalI = i;
+ originData.forEach(
+ (x, y) ->
+ y.removeIf(
+ timeValuePair ->
+ timeValuePair.getTimestamp() >= finalI * 100
+ && timeValuePair.getTimestamp() < finalI * 100 + 20));
+ resources.add(resource);
+ }
+ performer.setSourceFiles(resources);
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(resources, true);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new CompactionTaskSummary());
+ performer.perform();
+ Assert.assertTrue(targetResource.getTsFile().exists());
+ RestorableTsFileIOWriter checkWriter = new RestorableTsFileIOWriter(targetResource.getTsFile());
+ Assert.assertFalse(checkWriter.hasCrashed());
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
index 6ba310d439..68320ee32f 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
@@ -545,6 +545,10 @@ public class CompactionCheckerUtils {
List<TimeValuePair> expectedTimeValueList = expectedData.get(path);
List<TimeValuePair> actualTimeValueList = actualData.get(path);
+ if (actualTimeValueList == null) {
+ assertNull(expectedTimeValueList);
+ continue;
+ }
assertEquals(expectedTimeValueList.size(), actualTimeValueList.size());
for (int i = 0; i < expectedTimeValueList.size(); ++i) {