You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/01/30 09:54:30 UTC
[iotdb] branch master updated: [IOTDB-4684]Devices with the same name but different alignment properties are compacted into the wrong alignment property (#8820)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new fe7629694c [IOTDB-4684]Devices with the same name but different alignment properties are compacted into the wrong alignment property (#8820)
fe7629694c is described below
commit fe7629694c2d716d20583a498f4025146ae0303e
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Mon Jan 30 17:54:24 2023 +0800
[IOTDB-4684]Devices with the same name but different alignment properties are compacted into the wrong alignment property (#8820)
---
.../execute/utils/MultiTsFileDeviceIterator.java | 60 +++++++-----
.../utils/MultiTsFileDeviceIteratorTest.java | 105 +++++++++++++++++++++
.../iotdb/tsfile/read/TsFileDeviceIterator.java | 2 +-
3 files changed, 144 insertions(+), 23 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 785cb0fc6a..c10a2bb9f4 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -47,7 +47,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
public class MultiTsFileDeviceIterator implements AutoCloseable {
- private final List<TsFileResource> tsFileResources;
+
+ // sort from the newest to the oldest by version (Used by FastPerformer and ReadPointPerformer)
+ private final List<TsFileResource> tsFileResourcesSortedByDesc;
+
+ // sort from the oldest to the newest by version (Used by ReadChunkPerformer)
+ private List<TsFileResource> tsFileResourcesSortedByAsc;
private Map<TsFileResource, TsFileSequenceReader> readerMap = new HashMap<>();
private final Map<TsFileResource, TsFileDeviceIterator> deviceIteratorMap = new HashMap<>();
private final Map<TsFileResource, List<Modification>> modificationCache = new HashMap<>();
@@ -55,11 +60,14 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
/** Used for inner space compaction. */
public MultiTsFileDeviceIterator(List<TsFileResource> tsFileResources) throws IOException {
- this.tsFileResources = new ArrayList<>(tsFileResources);
+ this.tsFileResourcesSortedByDesc = new ArrayList<>(tsFileResources);
+ this.tsFileResourcesSortedByAsc = new ArrayList<>(tsFileResources);
// sort the files from the oldest to the newest
- Collections.sort(this.tsFileResources, TsFileResource::compareFileName);
+ Collections.sort(this.tsFileResourcesSortedByAsc, TsFileResource::compareFileName);
+ // sort the files from the newest to the oldest
+ Collections.sort(this.tsFileResourcesSortedByDesc, TsFileResource::compareFileNameByDesc);
try {
- for (TsFileResource tsFileResource : this.tsFileResources) {
+ for (TsFileResource tsFileResource : this.tsFileResourcesSortedByDesc) {
TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath());
readerMap.put(tsFileResource, reader);
deviceIteratorMap.put(tsFileResource, reader.getAllDevicesIteratorWithIsAligned());
@@ -77,11 +85,11 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
/** Used for cross space compaction with read point performer. */
public MultiTsFileDeviceIterator(
List<TsFileResource> seqResources, List<TsFileResource> unseqResources) throws IOException {
- this.tsFileResources = new ArrayList<>(seqResources);
- tsFileResources.addAll(unseqResources);
+ this.tsFileResourcesSortedByDesc = new ArrayList<>(seqResources);
+ tsFileResourcesSortedByDesc.addAll(unseqResources);
// sort the files from the newest to the oldest
- Collections.sort(this.tsFileResources, TsFileResource::compareFileNameByDesc);
- for (TsFileResource tsFileResource : tsFileResources) {
+ Collections.sort(this.tsFileResourcesSortedByDesc, TsFileResource::compareFileNameByDesc);
+ for (TsFileResource tsFileResource : tsFileResourcesSortedByDesc) {
TsFileSequenceReader reader =
FileReaderManager.getInstance().get(tsFileResource.getTsFilePath(), true);
readerMap.put(tsFileResource, reader);
@@ -95,12 +103,12 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
List<TsFileResource> unseqResources,
Map<TsFileResource, TsFileSequenceReader> readerMap)
throws IOException {
- this.tsFileResources = new ArrayList<>(seqResources);
- tsFileResources.addAll(unseqResources);
+ this.tsFileResourcesSortedByDesc = new ArrayList<>(seqResources);
+ tsFileResourcesSortedByDesc.addAll(unseqResources);
// sort tsfiles from the newest to the oldest
- Collections.sort(this.tsFileResources, TsFileResource::compareFileNameByDesc);
+ Collections.sort(this.tsFileResourcesSortedByDesc, TsFileResource::compareFileNameByDesc);
this.readerMap = readerMap;
- for (TsFileResource tsFileResource : tsFileResources) {
+ for (TsFileResource tsFileResource : tsFileResourcesSortedByDesc) {
TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getTsFilePath());
readerMap.put(tsFileResource, reader);
deviceIteratorMap.put(tsFileResource, reader.getAllDevicesIteratorWithIsAligned());
@@ -113,7 +121,8 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
hasNext =
hasNext
|| iterator.hasNext()
- || (iterator.current() != null && !iterator.current().equals(currentDevice));
+ || (iterator.current() != null
+ && !iterator.current().left.equals(currentDevice.left));
}
return hasNext;
}
@@ -126,9 +135,15 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
public Pair<String, Boolean> nextDevice() {
List<TsFileResource> toBeRemovedResources = new LinkedList<>();
Pair<String, Boolean> minDevice = null;
- for (TsFileResource resource : deviceIteratorMap.keySet()) {
+ // get the device from source files sorted from the newest to the oldest by version
+ for (TsFileResource resource : tsFileResourcesSortedByDesc) {
+ if (!deviceIteratorMap.containsKey(resource)) {
+ continue;
+ }
TsFileDeviceIterator deviceIterator = deviceIteratorMap.get(resource);
- if (deviceIterator.current() == null || deviceIterator.current().equals(currentDevice)) {
+ if (deviceIterator.current() == null
+ || deviceIterator.current().left.equals(currentDevice.left)) {
+ // if current file has same device with current device, then get its next device
if (deviceIterator.hasNext()) {
deviceIterator.next();
} else {
@@ -139,6 +154,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
}
}
if (minDevice == null || minDevice.left.compareTo(deviceIterator.current().left) > 0) {
+ // get the device that is minimal in lexicographical order according to the all files
minDevice = deviceIterator.current();
}
}
@@ -158,7 +174,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
public Map<String, MeasurementSchema> getAllSchemasOfCurrentDevice() throws IOException {
Map<String, MeasurementSchema> schemaMap = new ConcurrentHashMap<>();
// get schemas from the newest file to the oldest file
- for (TsFileResource resource : tsFileResources) {
+ for (TsFileResource resource : tsFileResourcesSortedByDesc) {
if (!deviceIteratorMap.containsKey(resource)
|| !deviceIteratorMap.get(resource).current().equals(currentDevice)) {
// if this tsfile has no more device or next device is not equals to the current device,
@@ -195,7 +211,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
getTimeseriesMetadataOffsetOfCurrentDevice() throws IOException {
Map<String, Map<TsFileResource, Pair<Long, Long>>> timeseriesMetadataOffsetMap =
new HashMap<>();
- for (TsFileResource resource : tsFileResources) {
+ for (TsFileResource resource : tsFileResourcesSortedByDesc) {
if (!deviceIteratorMap.containsKey(resource)
|| !deviceIteratorMap.get(resource).current().equals(currentDevice)) {
// if this tsfile has no more device or next device is not equals to the current device,
@@ -232,7 +248,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
getTimeseriesSchemaAndMetadataOffsetOfCurrentDevice() throws IOException {
Map<String, Pair<MeasurementSchema, Map<TsFileResource, Pair<Long, Long>>>>
timeseriesMetadataOffsetMap = new HashMap<>();
- for (TsFileResource resource : tsFileResources) {
+ for (TsFileResource resource : tsFileResourcesSortedByDesc) {
if (!deviceIteratorMap.containsKey(resource)
|| !deviceIteratorMap.get(resource).current().equals(currentDevice)) {
// if this tsfile has no more device or next device is not equals to the current device,
@@ -294,7 +310,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
LinkedList<Pair<TsFileSequenceReader, List<AlignedChunkMetadata>>> readerAndChunkMetadataList =
new LinkedList<>();
- for (TsFileResource tsFileResource : tsFileResources) {
+ for (TsFileResource tsFileResource : tsFileResourcesSortedByAsc) {
if (!deviceIteratorMap.containsKey(tsFileResource)) {
continue;
}
@@ -383,7 +399,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
this.device = device;
if (needDeserializeTimeseries) {
- for (TsFileResource resource : tsFileResources) {
+ for (TsFileResource resource : tsFileResourcesSortedByAsc) {
TsFileSequenceReader reader = readerMap.get(resource);
chunkMetadataIteratorMap.put(
resource, reader.getMeasurementChunkMetadataListMapIterator(device));
@@ -407,7 +423,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
private boolean collectSeries() {
String lastSeries = null;
List<String> tempCollectedSeries = new ArrayList<>();
- for (TsFileResource resource : tsFileResources) {
+ for (TsFileResource resource : tsFileResourcesSortedByAsc) {
TsFileSequenceReader reader = readerMap.get(resource);
Map<String, List<ChunkMetadata>> chunkMetadataListMap = chunkMetadataCacheMap.get(reader);
if (chunkMetadataListMap.size() == 0) {
@@ -490,7 +506,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
readerAndChunkMetadataForThisSeries = new LinkedList<>();
PartialPath path = new PartialPath(device, currentCompactingSeries);
- for (TsFileResource resource : tsFileResources) {
+ for (TsFileResource resource : tsFileResourcesSortedByAsc) {
TsFileSequenceReader reader = readerMap.get(resource);
Map<String, List<ChunkMetadata>> chunkMetadataListMap = chunkMetadataCacheMap.get(reader);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
index b8dc2a574c..9ab358f6e4 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/MultiTsFileDeviceIteratorTest.java
@@ -39,6 +39,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
+
public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
@Before
@@ -192,4 +194,107 @@ public class MultiTsFileDeviceIteratorTest extends AbstractCompactionTest {
}
Assert.assertEquals(30, deviceNum);
}
+
+ /**
+ * Create device with nonAligned property. Deleted it and create new device with same deviceID but
+ * aligned property. Check whether the deviceID and its property can be obtained correctly.
+ */
+ @Test
+ public void getDeletedDevicesWithSameNameFromDifferentFilesWithFourLayersInNodeTreeTest()
+ throws MetadataException, IOException, WriteProcessException {
+ TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3);
+ int oldAlignedDeviceOffset = TsFileGeneratorUtils.alignDeviceOffset;
+ TsFileGeneratorUtils.alignDeviceOffset = 0;
+ // create nonAligned device
+ registerTimeseriesInMManger(30, 5, false);
+ createFiles(3, 10, 5, 100, 0, 0, 50, 50, false, true);
+ createFiles(4, 5, 5, 100, 1000, 0, 50, 50, false, true);
+
+ // generate mods file, delete devices
+ List<String> seriesPaths = new ArrayList<>();
+ for (int i = 0; i < 15; i++) {
+ for (int j = 0; j < 5; j++) {
+ seriesPaths.add(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR + "s" + j);
+ }
+ }
+ generateModsFile(seriesPaths, seqResources, Long.MIN_VALUE, Long.MAX_VALUE);
+ generateModsFile(seriesPaths, unseqResources, Long.MIN_VALUE, Long.MAX_VALUE);
+ deleteTimeseriesInMManager(seriesPaths);
+
+ // create aligned device with the same deviceID
+ createFiles(2, 7, 5, 100, 1600, 1600, 50, 50, true, true);
+ createFiles(3, 30, 3, 100, 1000, 0, 50, 50, true, false);
+
+ // sort the deviceId in lexicographical order from small to large
+ List<String> deviceIds = new ArrayList<>();
+ for (int i = 0; i < 30; i++) {
+ deviceIds.add("root.testsg.d" + (i + TsFileGeneratorUtils.getAlignDeviceOffset()));
+ }
+ deviceIds.sort(String::compareTo);
+
+ int deviceNum = 0;
+ try (MultiTsFileDeviceIterator multiTsFileDeviceIterator =
+ new MultiTsFileDeviceIterator(seqResources, unseqResources)) {
+ while (multiTsFileDeviceIterator.hasNextDevice()) {
+ Pair<String, Boolean> deviceInfo = multiTsFileDeviceIterator.nextDevice();
+ Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left);
+ Assert.assertTrue(deviceInfo.right);
+ deviceNum++;
+ }
+ }
+ Assert.assertEquals(30, deviceNum);
+ TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset;
+ }
+
+ /**
+ * Create device with nonAligned property. Deleted it and create new device with same deviceID but
+ * aligned property. Check whether the deviceID and its property can be obtained correctly.
+ */
+ @Test
+ public void getDeletedDevicesWithSameNameFromSeqFilesWithFourLayersInNodeTreeTest()
+ throws MetadataException, IOException, WriteProcessException {
+ TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(3);
+ int oldAlignedDeviceOffset = TsFileGeneratorUtils.alignDeviceOffset;
+ TsFileGeneratorUtils.alignDeviceOffset = 0;
+ registerTimeseriesInMManger(30, 5, false);
+ createFiles(3, 10, 5, 100, 0, 0, 50, 50, false, true);
+ createFiles(4, 30, 5, 100, 1000, 0, 50, 50, false, true);
+
+ // generate mods file, delete d0 ~ d14
+ List<String> seriesPaths = new ArrayList<>();
+ for (int i = 0; i < 15; i++) {
+ for (int j = 0; j < 5; j++) {
+ seriesPaths.add(COMPACTION_TEST_SG + PATH_SEPARATOR + "d" + i + PATH_SEPARATOR + "s" + j);
+ }
+ }
+ generateModsFile(seriesPaths, seqResources, Long.MIN_VALUE, Long.MAX_VALUE);
+ generateModsFile(seriesPaths, unseqResources, Long.MIN_VALUE, Long.MAX_VALUE);
+ deleteTimeseriesInMManager(seriesPaths);
+
+ createFiles(2, 10, 5, 100, 2000, 2000, 50, 50, true, true);
+
+ // sort the deviceId in lexicographical order from small to large
+ List<String> deviceIds = new ArrayList<>();
+ for (int i = 0; i < 30; i++) {
+ deviceIds.add("root.testsg.d" + (i + TsFileGeneratorUtils.getAlignDeviceOffset()));
+ }
+ deviceIds.sort(String::compareTo);
+
+ int deviceNum = 0;
+ try (MultiTsFileDeviceIterator multiTsFileDeviceIterator =
+ new MultiTsFileDeviceIterator(seqResources)) {
+ while (multiTsFileDeviceIterator.hasNextDevice()) {
+ Pair<String, Boolean> deviceInfo = multiTsFileDeviceIterator.nextDevice();
+ Assert.assertEquals(deviceIds.get(deviceNum), deviceInfo.left);
+ if (Integer.parseInt(deviceInfo.left.substring(13)) < 10) {
+ Assert.assertTrue(deviceInfo.right);
+ } else {
+ Assert.assertFalse(deviceInfo.right);
+ }
+ deviceNum++;
+ }
+ }
+ Assert.assertEquals(30, deviceNum);
+ TsFileGeneratorUtils.alignDeviceOffset = oldAlignedDeviceOffset;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileDeviceIterator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileDeviceIterator.java
index a487d55b2a..76609747c3 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileDeviceIterator.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileDeviceIterator.java
@@ -80,7 +80,7 @@ public class TsFileDeviceIterator implements Iterator<Pair<String, Boolean>> {
}
Pair<String, long[]> startEndPair = queue.remove();
try {
- // get the first measurment node of this device, to know if the device is alignd
+ // get the first measurement node of this device, to know if the device is aligned
this.measurementNode =
MetadataIndexNode.deserializeFrom(
reader.readData(startEndPair.right[0], startEndPair.right[1]));