You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2026/09/16 04:01:05 UTC

(iotdb) 01/01: Fix aligned TVList page switch dropping duplicate values

This is an automated email from the ASF dual-hosted git repository.

shuwenwei pushed a commit to branch fix/aligned-tvlist-page-switch-duplicate-values
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 1a15a210e121799f2da8759ff8588c54bb0c5711
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Wed Sep 16 12:00:45 2026 +0800

    Fix aligned TVList page switch dropping duplicate values
---
 .../iotdb/db/utils/datastructure/TVList.java       |  5 +-
 .../memtable/AlignedTVListIteratorTest.java        | 86 ++++++++++++++++++++++
 2 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 506f82d9e10..d59da2575b0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -966,9 +966,10 @@ public abstract class TVList implements WALEntryValue {
           this.getQueryContext().getQueryStatistics().addFilteredRowsOfRowLevel(newIndex - index);
         }
         index = newIndex;
+        // If the cursor does not move, a duplicate-timestamp group prepared for the current
+        // position remains valid. Invalidate it only after the cursor actually advances.
+        probeNext = false;
       }
-
-      probeNext = false;
     }
 
     protected void prepareNext() {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
index c58eeb1bf60..ea2b43c8f8d 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
@@ -868,6 +868,92 @@ public class AlignedTVListIteratorTest {
             paginationController.getCurLimit(), paginationController.getCurOffset());
   }
 
+  @Test
+  public void testPageSwitchKeepsPreparedDuplicateTimestampValues() throws IOException {
+    AlignedTVList tvList =
+        AlignedTVList.newAlignedList(
+            Arrays.asList(TSDataType.INT64, TSDataType.BOOLEAN, TSDataType.BOOLEAN));
+    tvList.putAlignedValue(1, new Object[] {1L, true, false});
+    tvList.putAlignedValue(100, new Object[] {2L, null, false});
+    tvList.putAlignedValue(100, new Object[] {null, true, false});
+
+    Map<TVList, Integer> tvListMap = new LinkedHashMap<>();
+    tvListMap.put(tvList, tvList.rowCount());
+    AlignedReadOnlyMemChunk chunk =
+        new AlignedReadOnlyMemChunk(
+            fragmentInstanceContext,
+            Arrays.asList(0, 1, 2),
+            getMeasurementSchema(),
+            tvListMap,
+            Collections.emptyList(),
+            Arrays.asList(
+                Collections.emptyList(), Collections.emptyList(), Collections.emptyList()));
+    chunk.sortTvLists();
+    chunk.initChunkMetaFromTVListsWithFakeStatistics();
+
+    MemPointIterator iterator = chunk.createMemPointIterator(Ordering.ASC, null);
+    List<TimeValuePair> result = new ArrayList<>();
+    // These are fake-page boundaries for one MemChunk. The middle page is empty, but the
+    // shared iterator still receives its time range before its next page is read.
+    for (TimeRange pageRange :
+        Arrays.asList(new TimeRange(1, 33), new TimeRange(34, 66), new TimeRange(67, 100))) {
+      iterator.setCurrentPageTimeRange(pageRange);
+      while (iterator.hasNextTimeValuePair()) {
+        result.add(iterator.nextTimeValuePair());
+      }
+    }
+
+    Assert.assertEquals(2, result.size());
+    Assert.assertEquals(1L, result.get(0).getTimestamp());
+    Assert.assertEquals(1L, result.get(0).getValues()[0]);
+    Assert.assertEquals(100L, result.get(1).getTimestamp());
+    Assert.assertEquals(2L, result.get(1).getValues()[0]);
+    Assert.assertEquals(Boolean.TRUE, result.get(1).getValues()[1]);
+    Assert.assertEquals(Boolean.FALSE, result.get(1).getValues()[2]);
+  }
+
+  @Test
+  public void testPageSwitchKeepsPreparedDuplicateTimestampValuesDescending() throws IOException {
+    AlignedTVList tvList =
+        AlignedTVList.newAlignedList(
+            Arrays.asList(TSDataType.INT64, TSDataType.BOOLEAN, TSDataType.BOOLEAN));
+    tvList.putAlignedValue(1, new Object[] {null, true, false});
+    tvList.putAlignedValue(1, new Object[] {2L, null, false});
+    tvList.putAlignedValue(100, new Object[] {1L, true, false});
+
+    Map<TVList, Integer> tvListMap = new LinkedHashMap<>();
+    tvListMap.put(tvList, tvList.rowCount());
+    AlignedReadOnlyMemChunk chunk =
+        new AlignedReadOnlyMemChunk(
+            fragmentInstanceContext,
+            Arrays.asList(0, 1, 2),
+            getMeasurementSchema(),
+            tvListMap,
+            Collections.emptyList(),
+            Arrays.asList(
+                Collections.emptyList(), Collections.emptyList(), Collections.emptyList()));
+    chunk.sortTvLists();
+    chunk.initChunkMetaFromTVListsWithFakeStatistics();
+
+    MemPointIterator iterator = chunk.createMemPointIterator(Ordering.DESC, null);
+    List<TimeValuePair> result = new ArrayList<>();
+    for (TimeRange pageRange :
+        Arrays.asList(new TimeRange(67, 100), new TimeRange(34, 66), new TimeRange(1, 33))) {
+      iterator.setCurrentPageTimeRange(pageRange);
+      while (iterator.hasNextTimeValuePair()) {
+        result.add(iterator.nextTimeValuePair());
+      }
+    }
+
+    Assert.assertEquals(2, result.size());
+    Assert.assertEquals(100L, result.get(0).getTimestamp());
+    Assert.assertEquals(1L, result.get(0).getValues()[0]);
+    Assert.assertEquals(1L, result.get(1).getTimestamp());
+    Assert.assertEquals(2L, result.get(1).getValues()[0]);
+    Assert.assertEquals(Boolean.TRUE, result.get(1).getValues()[1]);
+    Assert.assertEquals(Boolean.FALSE, result.get(1).getValues()[2]);
+  }
+
   @Test
   public void testSkipTimeRange() throws QueryProcessException, IOException {
     List<Map<TVList, Integer>> list =