You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/12/17 07:20:39 UTC
(iotdb) branch rc/2.0.6 updated (be6857f2122 -> 914430555c4)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from be6857f2122 fixIT
new 20de362647e Optimize memtable region scan (#16883)
new d241ae66823 Bump ratis to apache 3.2.1 (#16801)
new 3753f6c7479 Switch to at.yawk.lz4:lz4-java:1.10.0 (#16871)
new 30d6845c2b8 Bump at.yawk.lz4:lz4-java from 1.10.0 to 1.10.1 (#16874)
new 914430555c4 Correct the dependencies.json
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE-binary | 4 +-
dependencies.json | 341 +++++++++----------
iotdb-client/session/pom.xml | 14 +
iotdb-core/confignode/pom.xml | 2 +-
iotdb-core/consensus/pom.xml | 14 +-
iotdb-core/datanode/pom.xml | 2 +-
.../db/storageengine/dataregion/DataRegion.java | 6 +-
.../dataregion/memtable/AbstractMemTable.java | 86 +++--
.../memtable/AlignedWritableMemChunk.java | 132 ++++++--
.../dataregion/memtable/IMemTable.java | 6 +-
.../dataregion/memtable/TsFileProcessor.java | 18 +-
.../dataregion/memtable/WritableMemChunk.java | 52 +--
.../memtable/WritableMemChunkRegionScanTest.java | 359 +++++++++++++++++++++
iotdb-core/node-commons/pom.xml | 2 +-
pom.xml | 32 +-
15 files changed, 794 insertions(+), 276 deletions(-)
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
(iotdb) 03/05: Switch to at.yawk.lz4:lz4-java:1.10.0 (#16871)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3753f6c74791242d44b585c03e3d01ca3dcbe5b8
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Dec 5 18:42:33 2025 +0800
Switch to at.yawk.lz4:lz4-java:1.10.0 (#16871)
(cherry picked from commit 2db7b21bce1f50eb4909d47b7fc4c9ca02dea7cf)
---
LICENSE-binary | 4 ++--
dependencies.json | 2 +-
iotdb-client/session/pom.xml | 14 ++++++++++++++
pom.xml | 18 +++++++++++++++---
4 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index b69bd29403a..83822262f73 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -213,7 +213,7 @@ conditions of the following licenses.
The binary distribution of this product bundles these dependencies under the
following license. See licenses/ for text of these licenses.
-Apache Software Foundation License 2.0
+Apache License 2.0
--------------------------------------
commons-cli:commons-cli:1.5.0
commons-codec:commons-codec:1.16.1
@@ -227,7 +227,7 @@ com.fasterxml.jackson.core:jackson-annotations:2.15.4
com.fasterxml.jackson.core:jackson-core:2.15.4
com.fasterxml.jackson.core:jackson-databind:2.15.4
jakarta.inject:jakarta.inject:2.6.1
-org.lz4:lz4-java:1.8.0
+at.yawk.lz4:lz4-java:1.10.0
com.github.stephenc.jcip:jcip-annotations:1.0-1
com.github.ben-manes.caffeine:caffeine:2.9.3
org.eclipse.jetty:jetty-http:9.4.56.v20240826
diff --git a/dependencies.json b/dependencies.json
index 6553a8dfe3e..dd439e372d4 100644
--- a/dependencies.json
+++ b/dependencies.json
@@ -1,5 +1,6 @@
{
"dependencies": [
+ "at.yawk.lz4:lz4-java",
"cglib:cglib",
"ch.qos.logback:logback-classic",
"ch.qos.logback:logback-core",
@@ -155,7 +156,6 @@
"org.jline:jline",
"org.jvnet.mimepull:mimepull",
"org.latencyutils:LatencyUtils",
- "org.lz4:lz4-java",
"org.ops4j.pax.jdbc:pax-jdbc-common",
"org.osgi:osgi.cmpn",
"org.osgi:osgi.core",
diff --git a/iotdb-client/session/pom.xml b/iotdb-client/session/pom.xml
index 733ca0c2bb5..449081ad360 100644
--- a/iotdb-client/session/pom.xml
+++ b/iotdb-client/session/pom.xml
@@ -59,6 +59,11 @@
<artifactId>tsfile</artifactId>
<version>${tsfile.version}</version>
</dependency>
+ <dependency>
+ <groupId>at.yawk.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
<artifactId>iotdb-thrift</artifactId>
@@ -121,6 +126,15 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <usedDependencies>
+ <usedDependency>at.yawk.lz4:lz4-java</usedDependency>
+ </usedDependencies>
+ </configuration>
+ </plugin>
</plugins>
</build>
<profiles>
diff --git a/pom.xml b/pom.xml
index 95e340601d4..7b26e1dcd0e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,8 +117,8 @@
<jtransforms.version>3.1</jtransforms.version>
<junit.version>4.13.2</junit.version>
<!-- This was the last version to support Java 8 -->
- <logback.version>1.3.15</logback.version>
- <lz4-java.version>1.8.0</lz4-java.version>
+ <logback.version>1.3.16</logback.version>
+ <lz4-java.version>1.10.0</lz4-java.version>
<maven.assembly.version>3.6.0</maven.assembly.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
@@ -318,6 +318,18 @@
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tsfile</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>${tsfile.version}</version>
+ <!-- Once tsfile switch to at.yawk.lz4:lz4-java, remove the following part -->
+ <exclusions>
+ <exclusion>
+ <groupId>org.lz4</groupId>
+ <artifactId>lz4-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.ratis</groupId>
<artifactId>ratis-server</artifactId>
@@ -509,7 +521,7 @@
<version>${zstd-jni.version}</version>
</dependency>
<dependency>
- <groupId>org.lz4</groupId>
+ <groupId>at.yawk.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>${lz4-java.version}</version>
</dependency>
(iotdb) 02/05: Bump ratis to apache 3.2.1 (#16801)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d241ae66823e76633d8514fe09abddbfde4eae11
Author: Jiang Tian <jt...@163.com>
AuthorDate: Wed Nov 26 09:35:54 2025 +0800
Bump ratis to apache 3.2.1 (#16801)
* Bump ratis to apache 3.2.1
* fix
(cherry picked from commit 1c6a2b02ab9473c1e3e38214317b4d398707021b)
---
dependencies.json | 335 ++++++++++++++++++++--------------------
iotdb-core/confignode/pom.xml | 2 +-
iotdb-core/consensus/pom.xml | 14 +-
iotdb-core/datanode/pom.xml | 2 +-
iotdb-core/node-commons/pom.xml | 2 +-
pom.xml | 14 +-
6 files changed, 186 insertions(+), 183 deletions(-)
diff --git a/dependencies.json b/dependencies.json
index 1f8f06da726..6553a8dfe3e 100644
--- a/dependencies.json
+++ b/dependencies.json
@@ -1,168 +1,171 @@
{
- "dependencies": [
- "cglib:cglib",
- "ch.qos.logback:logback-classic",
- "ch.qos.logback:logback-core",
- "ch.qos.reload4j:reload4j",
- "com.digitalpetri.fsm:strict-machine",
- "com.digitalpetri.netty:netty-channel-fsm",
- "com.fasterxml.jackson.core:jackson-annotations",
- "com.fasterxml.jackson.core:jackson-core",
- "com.fasterxml.jackson.core:jackson-databind",
- "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml",
- "com.fasterxml.jackson.datatype:jackson-datatype-jsr310",
- "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base",
- "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider",
- "com.fasterxml.jackson.module:jackson-module-jaxb-annotations",
- "com.github.ben-manes.caffeine:caffeine",
- "com.github.luben:zstd-jni",
- "com.github.stephenc.jcip:jcip-annotations",
- "com.github.wendykierp:JTransforms",
- "com.google.code.findbugs:jsr305",
- "com.google.code.gson:gson",
- "com.google.errorprone:error_prone_annotations",
- "com.google.guava:failureaccess",
- "com.google.guava:guava",
- "com.google.guava:listenablefuture",
- "com.google.j2objc:j2objc-annotations",
- "com.h2database:h2-mvstore",
- "com.lmax:disruptor",
- "com.nimbusds:content-type",
- "com.nimbusds:lang-tag",
- "com.nimbusds:nimbus-jose-jwt",
- "com.nimbusds:oauth2-oidc-sdk",
- "com.sun.istack:istack-commons-runtime",
- "com.zaxxer:HikariCP",
- "commons-cli:commons-cli",
- "commons-codec:commons-codec",
- "commons-io:commons-io",
- "commons-logging:commons-logging",
- "io.airlift:airline",
- "io.airlift:concurrent",
- "io.airlift:log",
- "io.airlift:units",
- "io.dropwizard.metrics:metrics-core",
- "io.jsonwebtoken:jjwt-api",
- "io.micrometer:micrometer-commons",
- "io.micrometer:micrometer-core",
- "io.micrometer:micrometer-observation",
- "com.github.moquette-io.moquette:moquette-broker",
- "io.netty:netty-buffer",
- "io.netty:netty-codec",
- "io.netty:netty-codec-dns",
- "io.netty:netty-codec-http",
- "io.netty:netty-codec-http2",
- "io.netty:netty-codec-mqtt",
- "io.netty:netty-codec-socks",
- "io.netty:netty-common",
- "io.netty:netty-handler",
- "io.netty:netty-handler-proxy",
- "io.netty:netty-resolver",
- "io.netty:netty-resolver-dns",
- "io.netty:netty-resolver-dns-classes-macos",
- "io.netty:netty-resolver-dns-native-macos",
- "io.netty:netty-tcnative-boringssl-static",
- "io.netty:netty-tcnative-classes",
- "io.netty:netty-transport",
- "io.netty:netty-transport-classes-epoll",
- "io.netty:netty-transport-native-epoll",
- "io.netty:netty-transport-native-unix-common",
- "io.netty:netty-transport-classes-kqueue",
- "io.netty:netty-transport-native-kqueue",
- "io.projectreactor:reactor-core",
- "io.projectreactor.netty:reactor-netty-core",
- "io.projectreactor.netty:reactor-netty-http",
- "io.swagger:swagger-annotations",
- "io.swagger:swagger-core",
- "io.swagger:swagger-jaxrs",
- "io.swagger:swagger-models",
- "jakarta.activation:jakarta.activation-api",
- "jakarta.annotation:jakarta.annotation-api",
- "jakarta.servlet:jakarta.servlet-api",
- "jakarta.validation:jakarta.validation-api",
- "jakarta.ws.rs:jakarta.ws.rs-api",
- "jakarta.xml.bind:jakarta.xml.bind-api",
- "net.java.dev.jna:jna",
- "net.java.dev.jna:jna-platform",
- "net.minidev:accessors-smart",
- "net.minidev:json-smart",
- "org.antlr:antlr4-runtime",
- "org.apache.commons:commons-collections4",
- "org.apache.commons:commons-csv",
- "org.apache.commons:commons-jexl3",
- "org.apache.commons:commons-lang3",
- "org.apache.commons:commons-math3",
- "org.apache.commons:commons-pool2",
- "org.apache.httpcomponents:httpclient",
- "org.apache.httpcomponents:httpcore",
- "com.timecho.ratis:ratis-client",
- "com.timecho.ratis:ratis-common",
- "com.timecho.ratis:ratis-grpc",
- "com.timecho.ratis:ratis-metrics-api",
- "com.timecho.ratis:ratis-proto",
- "com.timecho.ratis:ratis-server",
- "com.timecho.ratis:ratis-server-api",
- "org.apache.ratis:ratis-thirdparty-misc",
- "org.apache.thrift:libthrift",
- "org.apache.tsfile:common",
- "org.apache.tsfile:tsfile",
- "org.bouncycastle:bcpkix-jdk18on",
- "org.bouncycastle:bcprov-jdk18on",
- "org.bouncycastle:bcutil-jdk18on",
- "org.checkerframework:checker-qual",
- "org.eclipse.collections:eclipse-collections",
- "org.eclipse.collections:eclipse-collections-api",
- "org.eclipse.jetty:jetty-http",
- "org.eclipse.jetty:jetty-io",
- "org.eclipse.jetty:jetty-security",
- "org.eclipse.jetty:jetty-server",
- "org.eclipse.jetty:jetty-servlet",
- "org.eclipse.jetty:jetty-util",
- "org.eclipse.jetty:jetty-util-ajax",
- "org.eclipse.milo:bsd-core",
- "org.eclipse.milo:bsd-generator",
- "org.eclipse.milo:sdk-client",
- "org.eclipse.milo:sdk-core",
- "org.eclipse.milo:sdk-server",
- "org.eclipse.milo:stack-client",
- "org.eclipse.milo:stack-core",
- "org.eclipse.milo:stack-server",
- "org.fusesource.hawtbuf:hawtbuf",
- "org.fusesource.hawtdispatch:hawtdispatch",
- "org.fusesource.hawtdispatch:hawtdispatch-transport",
- "org.fusesource.mqtt-client:mqtt-client",
- "org.glassfish.hk2:hk2-api",
- "org.glassfish.hk2:hk2-locator",
- "org.glassfish.hk2:hk2-utils",
- "org.glassfish.hk2:osgi-resource-locator",
- "org.glassfish.hk2.external:aopalliance-repackaged",
- "org.glassfish.hk2.external:jakarta.inject",
- "org.glassfish.jaxb:jaxb-runtime",
- "org.glassfish.jaxb:txw2",
- "org.glassfish.jersey.containers:jersey-container-servlet-core",
- "org.glassfish.jersey.core:jersey-client",
- "org.glassfish.jersey.core:jersey-common",
- "org.glassfish.jersey.core:jersey-server",
- "org.glassfish.jersey.inject:jersey-hk2",
- "org.glassfish.jersey.media:jersey-media-multipart",
- "org.hdrhistogram:HdrHistogram",
- "org.java-websocket:Java-WebSocket",
- "org.javassist:javassist",
- "org.jline:jline",
- "org.jvnet.mimepull:mimepull",
- "org.latencyutils:LatencyUtils",
- "org.lz4:lz4-java",
- "org.ops4j.pax.jdbc:pax-jdbc-common",
- "org.osgi:osgi.cmpn",
- "org.osgi:osgi.core",
- "org.ow2.asm:asm",
- "org.reactivestreams:reactive-streams",
- "org.reflections:reflections",
- "org.slf4j:slf4j-api",
- "org.slf4j:slf4j-reload4j",
- "org.tukaani:xz",
- "org.xerial.snappy:snappy-java",
- "org.yaml:snakeyaml",
- "pl.edu.icm:JLargeArrays"
- ]
+ "dependencies": [
+ "cglib:cglib",
+ "ch.qos.logback:logback-classic",
+ "ch.qos.logback:logback-core",
+ "ch.qos.reload4j:reload4j",
+ "com.digitalpetri.fsm:strict-machine",
+ "com.digitalpetri.netty:netty-channel-fsm",
+ "com.fasterxml.jackson.core:jackson-annotations",
+ "com.fasterxml.jackson.core:jackson-core",
+ "com.fasterxml.jackson.core:jackson-databind",
+ "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml",
+ "com.fasterxml.jackson.datatype:jackson-datatype-jsr310",
+ "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base",
+ "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider",
+ "com.fasterxml.jackson.module:jackson-module-jaxb-annotations",
+ "com.github.ben-manes.caffeine:caffeine",
+ "com.github.luben:zstd-jni",
+ "com.github.moquette-io.moquette:moquette-broker",
+ "com.github.stephenc.jcip:jcip-annotations",
+ "com.github.wendykierp:JTransforms",
+ "com.google.code.findbugs:jsr305",
+ "com.google.code.gson:gson",
+ "com.google.errorprone:error_prone_annotations",
+ "com.google.guava:failureaccess",
+ "com.google.guava:guava",
+ "com.google.guava:listenablefuture",
+ "com.google.j2objc:j2objc-annotations",
+ "com.h2database:h2-mvstore",
+ "com.nimbusds:content-type",
+ "com.nimbusds:lang-tag",
+ "com.nimbusds:nimbus-jose-jwt",
+ "com.nimbusds:oauth2-oidc-sdk",
+ "com.sun.istack:istack-commons-runtime",
+ "org.apache.ratis:ratis-client",
+ "org.apache.ratis:ratis-common",
+ "org.apache.ratis:ratis-grpc",
+ "org.apache.ratis:ratis-metrics-api",
+ "org.apache.ratis:ratis-proto",
+ "org.apache.ratis:ratis-server",
+ "org.apache.ratis:ratis-server-api",
+ "com.zaxxer:HikariCP",
+ "commons-cli:commons-cli",
+ "commons-codec:commons-codec",
+ "commons-codec:commons-codec",
+ "commons-logging:commons-logging",
+ "io.airlift:airline",
+ "io.airlift:units",
+ "io.dropwizard.metrics:metrics-core",
+ "io.jsonwebtoken:jjwt-api",
+ "io.micrometer:micrometer-commons",
+ "io.micrometer:micrometer-core",
+ "io.micrometer:micrometer-observation",
+ "io.netty:netty-buffer",
+ "io.netty:netty-codec",
+ "io.netty:netty-codec-dns",
+ "io.netty:netty-codec-http",
+ "io.netty:netty-codec-http2",
+ "io.netty:netty-codec-mqtt",
+ "io.netty:netty-codec-socks",
+ "io.netty:netty-common",
+ "io.netty:netty-handler",
+ "io.netty:netty-handler-proxy",
+ "io.netty:netty-resolver",
+ "io.netty:netty-resolver-dns",
+ "io.netty:netty-resolver-dns-classes-macos",
+ "io.netty:netty-resolver-dns-native-macos",
+ "io.netty:netty-tcnative-boringssl-static",
+ "io.netty:netty-tcnative-boringssl-static",
+ "io.netty:netty-tcnative-boringssl-static",
+ "io.netty:netty-tcnative-boringssl-static",
+ "io.netty:netty-tcnative-boringssl-static",
+ "io.netty:netty-tcnative-classes",
+ "io.netty:netty-transport",
+ "io.netty:netty-transport-classes-epoll",
+ "io.netty:netty-transport-classes-kqueue",
+ "io.netty:netty-transport-native-epoll",
+ "io.netty:netty-transport-native-epoll",
+ "io.netty:netty-transport-native-kqueue",
+ "io.netty:netty-transport-native-kqueue",
+ "io.netty:netty-transport-native-unix-common",
+ "io.projectreactor:reactor-core",
+ "io.projectreactor.netty:reactor-netty-core",
+ "io.projectreactor.netty:reactor-netty-http",
+ "io.swagger:swagger-annotations",
+ "io.swagger:swagger-core",
+ "io.swagger:swagger-jaxrs",
+ "io.swagger:swagger-models",
+ "jakarta.activation:jakarta.activation-api",
+ "jakarta.annotation:jakarta.annotation-api",
+ "jakarta.servlet:jakarta.servlet-api",
+ "jakarta.validation:jakarta.validation-api",
+ "jakarta.ws.rs:jakarta.ws.rs-api",
+ "jakarta.xml.bind:jakarta.xml.bind-api",
+ "net.java.dev.jna:jna",
+ "net.java.dev.jna:jna-platform",
+ "net.minidev:accessors-smart",
+ "net.minidev:json-smart",
+ "org.antlr:antlr4-runtime",
+ "org.apache.commons:commons-csv",
+ "org.apache.commons:commons-jexl3",
+ "org.apache.commons:commons-lang3",
+ "org.apache.commons:commons-lang3",
+ "org.apache.commons:commons-math3",
+ "org.apache.commons:commons-pool2",
+ "org.apache.httpcomponents:httpclient",
+ "org.apache.httpcomponents:httpcore",
+ "org.apache.ratis:ratis-thirdparty-misc",
+ "org.apache.thrift:libthrift",
+ "org.apache.tsfile:common",
+ "org.apache.tsfile:tsfile",
+ "org.bouncycastle:bcpkix-jdk18on",
+ "org.bouncycastle:bcprov-jdk18on",
+ "org.bouncycastle:bcutil-jdk18on",
+ "org.checkerframework:checker-qual",
+ "org.checkerframework:checker-qual",
+ "org.eclipse.collections:eclipse-collections",
+ "org.eclipse.collections:eclipse-collections-api",
+ "org.eclipse.jetty:jetty-http",
+ "org.eclipse.jetty:jetty-io",
+ "org.eclipse.jetty:jetty-security",
+ "org.eclipse.jetty:jetty-server",
+ "org.eclipse.jetty:jetty-servlet",
+ "org.eclipse.jetty:jetty-util",
+ "org.eclipse.jetty:jetty-util-ajax",
+ "org.eclipse.milo:bsd-core",
+ "org.eclipse.milo:bsd-generator",
+ "org.eclipse.milo:sdk-client",
+ "org.eclipse.milo:sdk-core",
+ "org.eclipse.milo:sdk-server",
+ "org.eclipse.milo:stack-client",
+ "org.eclipse.milo:stack-core",
+ "org.eclipse.milo:stack-server",
+ "org.fusesource.hawtbuf:hawtbuf",
+ "org.fusesource.hawtdispatch:hawtdispatch",
+ "org.fusesource.hawtdispatch:hawtdispatch-transport",
+ "org.fusesource.mqtt-client:mqtt-client",
+ "org.glassfish.hk2:hk2-api",
+ "org.glassfish.hk2:hk2-locator",
+ "org.glassfish.hk2:hk2-utils",
+ "org.glassfish.hk2:osgi-resource-locator",
+ "org.glassfish.hk2.external:aopalliance-repackaged",
+ "org.glassfish.hk2.external:jakarta.inject",
+ "org.glassfish.jaxb:jaxb-runtime",
+ "org.glassfish.jaxb:txw2",
+ "org.glassfish.jersey.containers:jersey-container-servlet-core",
+ "org.glassfish.jersey.core:jersey-client",
+ "org.glassfish.jersey.core:jersey-common",
+ "org.glassfish.jersey.core:jersey-server",
+ "org.glassfish.jersey.inject:jersey-hk2",
+ "org.glassfish.jersey.media:jersey-media-multipart",
+ "org.hdrhistogram:HdrHistogram",
+ "org.java-websocket:Java-WebSocket",
+ "org.javassist:javassist",
+ "org.jline:jline",
+ "org.jvnet.mimepull:mimepull",
+ "org.latencyutils:LatencyUtils",
+ "org.lz4:lz4-java",
+ "org.ops4j.pax.jdbc:pax-jdbc-common",
+ "org.osgi:osgi.cmpn",
+ "org.osgi:osgi.core",
+ "org.ow2.asm:asm",
+ "org.reactivestreams:reactive-streams",
+ "org.slf4j:slf4j-api",
+ "org.slf4j:slf4j-reload4j",
+ "org.tukaani:xz",
+ "org.xerial.snappy:snappy-java",
+ "org.yaml:snakeyaml",
+ "pl.edu.icm:JLargeArrays"
+ ]
}
diff --git a/iotdb-core/confignode/pom.xml b/iotdb-core/confignode/pom.xml
index 14d0d71cd35..5e77b0eaa5f 100644
--- a/iotdb-core/confignode/pom.xml
+++ b/iotdb-core/confignode/pom.xml
@@ -117,7 +117,7 @@
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
</dependency>
<dependency>
diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml
index 4d3441dfcd5..aaa5294303c 100644
--- a/iotdb-core/consensus/pom.xml
+++ b/iotdb-core/consensus/pom.xml
@@ -67,31 +67,31 @@
<version>2.0.6-SNAPSHOT</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-server</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-server-api</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-client</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-grpc</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-proto</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-metrics-api</artifactId>
</dependency>
<dependency>
diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml
index 5d5801a2158..e5bbe5c2152 100644
--- a/iotdb-core/datanode/pom.xml
+++ b/iotdb-core/datanode/pom.xml
@@ -164,7 +164,7 @@
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
</dependency>
<dependency>
diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml
index cbc07cdf181..f4dee08df37 100644
--- a/iotdb-core/node-commons/pom.xml
+++ b/iotdb-core/node-commons/pom.xml
@@ -191,7 +191,7 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
</dependency>
</dependencies>
diff --git a/pom.xml b/pom.xml
index 1d117ccab0e..95e340601d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -319,37 +319,37 @@
<version>${commons-io.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-server</artifactId>
<version>${ratis.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-server-api</artifactId>
<version>${ratis.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
<version>${ratis.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-client</artifactId>
<version>${ratis.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-grpc</artifactId>
<version>${ratis.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-proto</artifactId>
<version>${ratis.version}</version>
</dependency>
<dependency>
- <groupId>com.timecho.ratis</groupId>
+ <groupId>org.apache.ratis</groupId>
<artifactId>ratis-metrics-api</artifactId>
<version>${ratis.version}</version>
</dependency>
(iotdb) 05/05: Correct the dependencies.json
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 914430555c4c418a308998fb26330a94a425a06e
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Dec 17 15:15:19 2025 +0800
Correct the dependencies.json
---
dependencies.json | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/dependencies.json b/dependencies.json
index dd439e372d4..7da90c749e0 100644
--- a/dependencies.json
+++ b/dependencies.json
@@ -28,24 +28,20 @@
"com.google.guava:listenablefuture",
"com.google.j2objc:j2objc-annotations",
"com.h2database:h2-mvstore",
+ "com.lmax:disruptor",
"com.nimbusds:content-type",
"com.nimbusds:lang-tag",
"com.nimbusds:nimbus-jose-jwt",
"com.nimbusds:oauth2-oidc-sdk",
"com.sun.istack:istack-commons-runtime",
- "org.apache.ratis:ratis-client",
- "org.apache.ratis:ratis-common",
- "org.apache.ratis:ratis-grpc",
- "org.apache.ratis:ratis-metrics-api",
- "org.apache.ratis:ratis-proto",
- "org.apache.ratis:ratis-server",
- "org.apache.ratis:ratis-server-api",
"com.zaxxer:HikariCP",
"commons-cli:commons-cli",
"commons-codec:commons-codec",
- "commons-codec:commons-codec",
+ "commons-io:commons-io",
"commons-logging:commons-logging",
"io.airlift:airline",
+ "io.airlift:concurrent",
+ "io.airlift:log",
"io.airlift:units",
"io.dropwizard.metrics:metrics-core",
"io.jsonwebtoken:jjwt-api",
@@ -56,6 +52,7 @@
"io.netty:netty-codec",
"io.netty:netty-codec-dns",
"io.netty:netty-codec-http",
+ "io.netty:netty-codec-http",
"io.netty:netty-codec-http2",
"io.netty:netty-codec-mqtt",
"io.netty:netty-codec-socks",
@@ -98,14 +95,21 @@
"net.minidev:accessors-smart",
"net.minidev:json-smart",
"org.antlr:antlr4-runtime",
+ "org.apache.commons:commons-collections4",
"org.apache.commons:commons-csv",
"org.apache.commons:commons-jexl3",
"org.apache.commons:commons-lang3",
- "org.apache.commons:commons-lang3",
"org.apache.commons:commons-math3",
"org.apache.commons:commons-pool2",
"org.apache.httpcomponents:httpclient",
"org.apache.httpcomponents:httpcore",
+ "org.apache.ratis:ratis-client",
+ "org.apache.ratis:ratis-common",
+ "org.apache.ratis:ratis-grpc",
+ "org.apache.ratis:ratis-metrics-api",
+ "org.apache.ratis:ratis-proto",
+ "org.apache.ratis:ratis-server",
+ "org.apache.ratis:ratis-server-api",
"org.apache.ratis:ratis-thirdparty-misc",
"org.apache.thrift:libthrift",
"org.apache.tsfile:common",
@@ -114,7 +118,6 @@
"org.bouncycastle:bcprov-jdk18on",
"org.bouncycastle:bcutil-jdk18on",
"org.checkerframework:checker-qual",
- "org.checkerframework:checker-qual",
"org.eclipse.collections:eclipse-collections",
"org.eclipse.collections:eclipse-collections-api",
"org.eclipse.jetty:jetty-http",
@@ -161,6 +164,7 @@
"org.osgi:osgi.core",
"org.ow2.asm:asm",
"org.reactivestreams:reactive-streams",
+ "org.reflections:reflections",
"org.slf4j:slf4j-api",
"org.slf4j:slf4j-reload4j",
"org.tukaani:xz",
@@ -168,4 +172,4 @@
"org.yaml:snakeyaml",
"pl.edu.icm:JLargeArrays"
]
-}
+}
\ No newline at end of file
(iotdb) 01/05: Optimize memtable region scan (#16883)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 20de362647e9f90e6044167f4c608679a3225bd6
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Tue Dec 9 19:10:23 2025 +0800
Optimize memtable region scan (#16883)
(cherry picked from commit a899c48a09c94de1c6bfd7730d013b5d12b1a0b9)
---
.../db/storageengine/dataregion/DataRegion.java | 6 +-
.../dataregion/memtable/AbstractMemTable.java | 86 +++--
.../memtable/AlignedWritableMemChunk.java | 132 ++++++--
.../dataregion/memtable/IMemTable.java | 6 +-
.../dataregion/memtable/TsFileProcessor.java | 18 +-
.../dataregion/memtable/WritableMemChunk.java | 52 +--
.../memtable/WritableMemChunkRegionScanTest.java | 359 +++++++++++++++++++++
7 files changed, 572 insertions(+), 87 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index a6e04558af8..05d3042c1b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2319,7 +2319,8 @@ public class DataRegion implements IDataRegionForQuery {
} else {
tsFileResource
.getProcessor()
- .queryForSeriesRegionScanWithoutLock(partialPaths, context, fileScanHandles);
+ .queryForSeriesRegionScanWithoutLock(
+ partialPaths, context, fileScanHandles, globalTimeFilter);
}
}
return fileScanHandles;
@@ -2396,7 +2397,8 @@ public class DataRegion implements IDataRegionForQuery {
} else {
tsFileResource
.getProcessor()
- .queryForDeviceRegionScanWithoutLock(devicePathsToContext, context, fileScanHandles);
+ .queryForDeviceRegionScanWithoutLock(
+ devicePathsToContext, context, fileScanHandles, globalTimeFilter);
}
}
return fileScanHandles;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index a4b235650cf..db49fef2bc3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -68,6 +68,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -441,7 +442,8 @@ public abstract class AbstractMemTable implements IMemTable {
long ttlLowerBound,
Map<String, List<IChunkMetadata>> chunkMetaDataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
- List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
+ List<Pair<ModEntry, IMemTable>> modsToMemTabled,
+ Filter globalTimeFilter) {
IDeviceID deviceID = fullPath.getDeviceId();
if (fullPath instanceof NonAlignedFullPath) {
@@ -459,7 +461,12 @@ public abstract class AbstractMemTable implements IMemTable {
fullPath.getDeviceId(), measurementId, this, modsToMemTabled, ttlLowerBound);
}
getMemChunkHandleFromMemTable(
- deviceID, measurementId, chunkMetaDataMap, memChunkHandleMap, deletionList);
+ deviceID,
+ measurementId,
+ chunkMetaDataMap,
+ memChunkHandleMap,
+ deletionList,
+ globalTimeFilter);
} else {
// check If MemTable Contains this path
if (!memTableMap.containsKey(deviceID)) {
@@ -481,7 +488,8 @@ public abstract class AbstractMemTable implements IMemTable {
((AlignedFullPath) fullPath).getSchemaList(),
chunkMetaDataMap,
memChunkHandleMap,
- deletionList);
+ deletionList,
+ globalTimeFilter);
}
}
@@ -492,7 +500,8 @@ public abstract class AbstractMemTable implements IMemTable {
long ttlLowerBound,
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
- List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
+ List<Pair<ModEntry, IMemTable>> modsToMemTabled,
+ Filter globalTimeFilter) {
Map<IDeviceID, IWritableMemChunkGroup> memTableMap = getMemTableMap();
@@ -509,7 +518,8 @@ public abstract class AbstractMemTable implements IMemTable {
chunkMetadataMap,
memChunkHandleMap,
ttlLowerBound,
- modsToMemTabled);
+ modsToMemTabled,
+ globalTimeFilter);
} else {
getMemChunkHandleFromMemTable(
deviceID,
@@ -517,7 +527,8 @@ public abstract class AbstractMemTable implements IMemTable {
chunkMetadataMap,
memChunkHandleMap,
ttlLowerBound,
- modsToMemTabled);
+ modsToMemTabled,
+ globalTimeFilter);
}
}
@@ -526,24 +537,30 @@ public abstract class AbstractMemTable implements IMemTable {
String measurementId,
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
- List<TimeRange> deletionList) {
+ List<TimeRange> deletionList,
+ Filter globalTimeFilter) {
WritableMemChunk memChunk =
(WritableMemChunk) memTableMap.get(deviceID).getMemChunkMap().get(measurementId);
- long[] timestamps = memChunk.getFilteredTimestamp(deletionList);
+ if (memChunk == null) {
+ return;
+ }
+ Optional<Long> anySatisfiedTimestamp =
+ memChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
+ if (!anySatisfiedTimestamp.isPresent()) {
+ return;
+ }
+ long satisfiedTimestamp = anySatisfiedTimestamp.get();
chunkMetadataMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
.add(
- buildChunkMetaDataForMemoryChunk(
- measurementId,
- timestamps[0],
- timestamps[timestamps.length - 1],
- Collections.emptyList()));
+ buildFakeChunkMetaDataForFakeMemoryChunk(
+ measurementId, satisfiedTimestamp, satisfiedTimestamp, Collections.emptyList()));
memChunkHandleMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
- .add(new MemChunkHandleImpl(deviceID, measurementId, timestamps));
+ .add(new MemChunkHandleImpl(deviceID, measurementId, new long[] {satisfiedTimestamp}));
}
private void getMemAlignedChunkHandleFromMemTable(
@@ -551,7 +568,8 @@ public abstract class AbstractMemTable implements IMemTable {
List<IMeasurementSchema> schemaList,
Map<String, List<IChunkMetadata>> chunkMetadataList,
Map<String, List<IChunkHandle>> memChunkHandleMap,
- List<List<TimeRange>> deletionList) {
+ List<List<TimeRange>> deletionList,
+ Filter globalTimeFilter) {
AlignedWritableMemChunk alignedMemChunk =
((AlignedWritableMemChunkGroup) memTableMap.get(deviceID)).getAlignedMemChunk();
@@ -568,7 +586,11 @@ public abstract class AbstractMemTable implements IMemTable {
}
List<BitMap> bitMaps = new ArrayList<>();
- long[] timestamps = alignedMemChunk.getFilteredTimestamp(deletionList, bitMaps, true);
+ long[] timestamps =
+ alignedMemChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter);
+ if (timestamps.length == 0) {
+ return;
+ }
buildAlignedMemChunkHandle(
deviceID,
@@ -586,7 +608,8 @@ public abstract class AbstractMemTable implements IMemTable {
Map<String, List<IChunkMetadata>> chunkMetadataList,
Map<String, List<IChunkHandle>> memChunkHandleMap,
long ttlLowerBound,
- List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
+ List<Pair<ModEntry, IMemTable>> modsToMemTabled,
+ Filter globalTimeFilter) {
AlignedWritableMemChunk memChunk = writableMemChunkGroup.getAlignedMemChunk();
List<IMeasurementSchema> schemaList = memChunk.getSchemaList();
@@ -601,7 +624,11 @@ public abstract class AbstractMemTable implements IMemTable {
}
List<BitMap> bitMaps = new ArrayList<>();
- long[] timestamps = memChunk.getFilteredTimestamp(deletionList, bitMaps, true);
+ long[] timestamps =
+ memChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter);
+ if (timestamps.length == 0) {
+ return;
+ }
buildAlignedMemChunkHandle(
deviceID,
timestamps,
@@ -618,7 +645,8 @@ public abstract class AbstractMemTable implements IMemTable {
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
long ttlLowerBound,
- List<Pair<ModEntry, IMemTable>> modsToMemTabled) {
+ List<Pair<ModEntry, IMemTable>> modsToMemTabled,
+ Filter globalTimeFilter) {
for (Entry<String, IWritableMemChunk> entry :
writableMemChunkGroup.getMemChunkMap().entrySet()) {
@@ -632,18 +660,20 @@ public abstract class AbstractMemTable implements IMemTable {
ModificationUtils.constructDeletionList(
deviceID, measurementId, this, modsToMemTabled, ttlLowerBound);
}
- long[] timestamps = writableMemChunk.getFilteredTimestamp(deletionList);
+ Optional<Long> anySatisfiedTimestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
+ if (!anySatisfiedTimestamp.isPresent()) {
+ return;
+ }
+ long satisfiedTimestamp = anySatisfiedTimestamp.get();
chunkMetadataMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
.add(
- buildChunkMetaDataForMemoryChunk(
- measurementId,
- timestamps[0],
- timestamps[timestamps.length - 1],
- Collections.emptyList()));
+ buildFakeChunkMetaDataForFakeMemoryChunk(
+ measurementId, satisfiedTimestamp, satisfiedTimestamp, Collections.emptyList()));
memChunkHandleMap
.computeIfAbsent(measurementId, k -> new ArrayList<>())
- .add(new MemChunkHandleImpl(deviceID, measurementId, timestamps));
+ .add(new MemChunkHandleImpl(deviceID, measurementId, new long[] {satisfiedTimestamp}));
}
}
@@ -667,7 +697,7 @@ public abstract class AbstractMemTable implements IMemTable {
chunkMetadataList
.computeIfAbsent(measurement, k -> new ArrayList<>())
.add(
- buildChunkMetaDataForMemoryChunk(
+ buildFakeChunkMetaDataForFakeMemoryChunk(
measurement, startEndTime[0], startEndTime[1], deletion));
chunkHandleMap
.computeIfAbsent(measurement, k -> new ArrayList<>())
@@ -698,7 +728,7 @@ public abstract class AbstractMemTable implements IMemTable {
return new long[] {startTime, endTime};
}
- private IChunkMetadata buildChunkMetaDataForMemoryChunk(
+ private IChunkMetadata buildFakeChunkMetaDataForFakeMemoryChunk(
String measurement, long startTime, long endTime, List<TimeRange> deletionList) {
TimeStatistics timeStatistics = new TimeStatistics();
timeStatistics.setStartTime(startTime);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index d41ef855d36..2cab901ce6b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
@@ -53,6 +54,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
@@ -266,30 +268,99 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
return new Pair<>(reorderedColumnValues, reorderedBitMaps);
}
- private void filterDeletedTimeStamp(
+ public long[] getAnySatisfiedTimestamp(
+ List<List<TimeRange>> deletionList,
+ List<BitMap> bitMaps,
+ boolean ignoreAllNullRows,
+ Filter globalTimeFilter) {
+ BitMap columnHasNonNullValue = new BitMap(schemaList.size());
+ AtomicInteger hasNonNullValueColumnCount = new AtomicInteger(0);
+ Map<Long, BitMap> timestampWithBitmap = new TreeMap<>();
+
+ getAnySatisfiedTimestamp(
+ list,
+ deletionList,
+ ignoreAllNullRows,
+ timestampWithBitmap,
+ globalTimeFilter,
+ columnHasNonNullValue,
+ hasNonNullValueColumnCount);
+ for (int i = 0;
+ i < sortedList.size() && hasNonNullValueColumnCount.get() < schemaList.size();
+ i++) {
+ if (!ignoreAllNullRows && !timestampWithBitmap.isEmpty()) {
+ // count devices in table model
+ break;
+ }
+ getAnySatisfiedTimestamp(
+ sortedList.get(i),
+ deletionList,
+ ignoreAllNullRows,
+ timestampWithBitmap,
+ globalTimeFilter,
+ columnHasNonNullValue,
+ hasNonNullValueColumnCount);
+ }
+
+ long[] timestamps = new long[timestampWithBitmap.size()];
+ int idx = 0;
+ for (Map.Entry<Long, BitMap> entry : timestampWithBitmap.entrySet()) {
+ timestamps[idx++] = entry.getKey();
+ bitMaps.add(entry.getValue());
+ }
+ return timestamps;
+ }
+
+ private void getAnySatisfiedTimestamp(
AlignedTVList alignedTVList,
List<List<TimeRange>> valueColumnsDeletionList,
boolean ignoreAllNullRows,
- Map<Long, BitMap> timestampWithBitmap) {
+ Map<Long, BitMap> timestampWithBitmap,
+ Filter globalTimeFilter,
+ BitMap columnHasNonNullValue,
+ AtomicInteger hasNonNullValueColumnCount) {
+ if (globalTimeFilter != null
+ && !globalTimeFilter.satisfyStartEndTime(
+ alignedTVList.getMinTime(), alignedTVList.getMaxTime())) {
+ return;
+ }
BitMap allValueColDeletedMap = alignedTVList.getAllValueColDeletedMap();
-
int rowCount = alignedTVList.rowCount();
List<int[]> valueColumnDeleteCursor = new ArrayList<>();
if (valueColumnsDeletionList != null) {
valueColumnsDeletionList.forEach(x -> valueColumnDeleteCursor.add(new int[] {0}));
}
+ // example:
+ // globalTimeFilter:null, ignoreAllNullRows: true
+ // tvList:
+ // time s1 s2 s3
+ // 1 1 null null
+ // 2 null 1 null
+ // 2 1 1 null
+ // 3 1 null null
+ // 4 1 null 1
+ // timestampWithBitmap:
+ // timestamp: 1 bitmap: 011
+ // timestamp: 2 bitmap: 101
+ // timestamp: 4 bitmap: 110
for (int row = 0; row < rowCount; row++) {
// the row is deleted
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
continue;
}
long timestamp = alignedTVList.getTime(row);
+ if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp, null)) {
+ continue;
+ }
+
+ // Note that this method will only perform bitmap unmarking on the first occurrence of a
+ // non-null value in multiple timestamps for the same column.
+ BitMap currentRowNullValueBitmap = null;
- BitMap bitMap = new BitMap(schemaList.size());
for (int column = 0; column < schemaList.size(); column++) {
if (alignedTVList.isNullValue(alignedTVList.getValueIndex(row), column)) {
- bitMap.mark(column);
+ continue;
}
// skip deleted row
@@ -299,33 +370,44 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
timestamp,
valueColumnsDeletionList.get(column),
valueColumnDeleteCursor.get(column))) {
- bitMap.mark(column);
- }
-
- // skip all-null row
- if (ignoreAllNullRows && bitMap.isAllMarked()) {
continue;
}
- timestampWithBitmap.put(timestamp, bitMap);
+ if (!columnHasNonNullValue.isMarked(column)) {
+ hasNonNullValueColumnCount.incrementAndGet();
+ columnHasNonNullValue.mark(column);
+ currentRowNullValueBitmap =
+ currentRowNullValueBitmap != null
+ ? currentRowNullValueBitmap
+ : timestampWithBitmap.computeIfAbsent(
+ timestamp, k -> getAllMarkedBitmap(schemaList.size()));
+ currentRowNullValueBitmap.unmark(column);
+ }
}
- }
- }
- public long[] getFilteredTimestamp(
- List<List<TimeRange>> deletionList, List<BitMap> bitMaps, boolean ignoreAllNullRows) {
- Map<Long, BitMap> timestampWithBitmap = new TreeMap<>();
+ if (!ignoreAllNullRows) {
+ timestampWithBitmap.put(
+ timestamp,
+ currentRowNullValueBitmap != null
+ ? currentRowNullValueBitmap
+ : getAllMarkedBitmap(schemaList.size()));
+ return;
+ }
+ if (currentRowNullValueBitmap == null) {
+ continue;
+ }
+ // found new column with non-null value
+ timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
- filterDeletedTimeStamp(list, deletionList, ignoreAllNullRows, timestampWithBitmap);
- for (AlignedTVList alignedTVList : sortedList) {
- filterDeletedTimeStamp(alignedTVList, deletionList, ignoreAllNullRows, timestampWithBitmap);
+ if (hasNonNullValueColumnCount.get() == schemaList.size()) {
+ return;
+ }
}
+ }
- List<Long> filteredTimestamps = new ArrayList<>();
- for (Map.Entry<Long, BitMap> entry : timestampWithBitmap.entrySet()) {
- filteredTimestamps.add(entry.getKey());
- bitMaps.add(entry.getValue());
- }
- return filteredTimestamps.stream().mapToLong(Long::valueOf).toArray();
+ private BitMap getAllMarkedBitmap(int size) {
+ BitMap bitMap = new BitMap(size);
+ bitMap.markAll();
+ return bitMap;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index 6c6e09c5782..fd9ffe90b0a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -125,7 +125,8 @@ public interface IMemTable extends WALEntryValue {
long ttlLowerBound,
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
- List<Pair<ModEntry, IMemTable>> modsToMemtabled)
+ List<Pair<ModEntry, IMemTable>> modsToMemtabled,
+ Filter globalTimeFilter)
throws IOException, QueryProcessException, MetadataException;
void queryForDeviceRegionScan(
@@ -134,7 +135,8 @@ public interface IMemTable extends WALEntryValue {
long ttlLowerBound,
Map<String, List<IChunkMetadata>> chunkMetadataMap,
Map<String, List<IChunkHandle>> memChunkHandleMap,
- List<Pair<ModEntry, IMemTable>> modsToMemtabled)
+ List<Pair<ModEntry, IMemTable>> modsToMemtabled,
+ Filter globalTimeFilter)
throws IOException, QueryProcessException, MetadataException;
/** putBack all the memory resources. */
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index baeb27187fd..3913403a5de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1967,7 +1967,8 @@ public class TsFileProcessor {
public void queryForSeriesRegionScanWithoutLock(
List<IFullPath> pathList,
QueryContext queryContext,
- List<IFileScanHandle> fileScanHandlesForQuery) {
+ List<IFileScanHandle> fileScanHandlesForQuery,
+ Filter globalTimeFilter) {
long startTime = System.nanoTime();
try {
Map<IDeviceID, Map<String, List<IChunkHandle>>> deviceToMemChunkHandleMap = new HashMap<>();
@@ -1988,7 +1989,8 @@ public class TsFileProcessor {
timeLowerBound,
measurementToChunkMetaList,
measurementToChunkHandleList,
- modsToMemtable);
+ modsToMemtable,
+ globalTimeFilter);
}
if (workMemTable != null) {
workMemTable.queryForSeriesRegionScan(
@@ -1996,7 +1998,8 @@ public class TsFileProcessor {
timeLowerBound,
measurementToChunkMetaList,
measurementToChunkHandleList,
- null);
+ null,
+ globalTimeFilter);
}
IDeviceID deviceID = seriesPath.getDeviceId();
// Some memTable have been flushed already, so we need to get the chunk metadata from
@@ -2047,7 +2050,8 @@ public class TsFileProcessor {
public void queryForDeviceRegionScanWithoutLock(
Map<IDeviceID, DeviceContext> devicePathsToContext,
QueryContext queryContext,
- List<IFileScanHandle> fileScanHandlesForQuery) {
+ List<IFileScanHandle> fileScanHandlesForQuery,
+ Filter globalTimeFilter) {
long startTime = System.nanoTime();
try {
Map<IDeviceID, Map<String, List<IChunkHandle>>> deviceToMemChunkHandleMap = new HashMap<>();
@@ -2070,7 +2074,8 @@ public class TsFileProcessor {
timeLowerBound,
measurementToChunkMetadataList,
measurementToMemChunkHandleList,
- modsToMemtable);
+ modsToMemtable,
+ globalTimeFilter);
}
if (workMemTable != null) {
workMemTable.queryForDeviceRegionScan(
@@ -2079,7 +2084,8 @@ public class TsFileProcessor {
timeLowerBound,
measurementToChunkMetadataList,
measurementToMemChunkHandleList,
- null);
+ null,
+ globalTimeFilter);
}
buildChunkHandleForFlushedMemTable(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index 1f4078dfc7d..6c9fde0c533 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.write.UnSupportedDataTypeException;
@@ -45,10 +46,9 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
+import java.util.Optional;
import java.util.concurrent.BlockingQueue;
-import java.util.stream.Collectors;
import static org.apache.iotdb.db.utils.MemUtils.getBinarySize;
@@ -558,11 +558,30 @@ public class WritableMemChunk extends AbstractWritableMemChunk {
return sortedList;
}
- private void filterDeletedTimestamp(
- TVList tvlist, List<TimeRange> deletionList, List<Long> timestampList) {
- long lastTime = Long.MIN_VALUE;
+ public Optional<Long> getAnySatisfiedTimestamp(
+ List<TimeRange> deletionList, Filter globalTimeFilter) {
+ Optional<Long> anySatisfiedTimestamp =
+ getAnySatisfiedTimestamp(list, deletionList, globalTimeFilter);
+ if (anySatisfiedTimestamp.isPresent()) {
+ return anySatisfiedTimestamp;
+ }
+ for (TVList tvList : sortedList) {
+ anySatisfiedTimestamp = getAnySatisfiedTimestamp(tvList, deletionList, globalTimeFilter);
+ if (anySatisfiedTimestamp.isPresent()) {
+ break;
+ }
+ }
+ return anySatisfiedTimestamp;
+ }
+
+ private Optional<Long> getAnySatisfiedTimestamp(
+ TVList tvlist, List<TimeRange> deletionList, Filter globalTimeFilter) {
int[] deletionCursor = {0};
int rowCount = tvlist.rowCount();
+ if (globalTimeFilter != null
+ && !globalTimeFilter.satisfyStartEndTime(tvlist.getMinTime(), tvlist.getMaxTime())) {
+ return Optional.empty();
+ }
for (int i = 0; i < rowCount; i++) {
if (tvlist.getBitMap() != null && tvlist.isNullValue(tvlist.getValueIndex(i))) {
continue;
@@ -572,26 +591,11 @@ public class WritableMemChunk extends AbstractWritableMemChunk {
&& ModificationUtils.isPointDeleted(curTime, deletionList, deletionCursor)) {
continue;
}
-
- if (i == rowCount - 1 || curTime != lastTime) {
- timestampList.add(curTime);
+ if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime, null)) {
+ continue;
}
- lastTime = curTime;
+ return Optional.of(curTime);
}
- }
-
- public long[] getFilteredTimestamp(List<TimeRange> deletionList) {
- List<Long> timestampList = new ArrayList<>();
- filterDeletedTimestamp(list, deletionList, timestampList);
- for (TVList tvList : sortedList) {
- filterDeletedTimestamp(tvList, deletionList, timestampList);
- }
-
- // remove duplicated time
- List<Long> distinctTimestamps = timestampList.stream().distinct().collect(Collectors.toList());
- // sort timestamps
- long[] filteredTimestamps = distinctTimestamps.stream().mapToLong(Long::longValue).toArray();
- Arrays.sort(filteredTimestamps);
- return filteredTimestamps;
+ return Optional.empty();
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
new file mode 100644
index 00000000000..3967409db52
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.memtable;
+
+import org.apache.iotdb.commons.path.AlignedFullPath;
+import org.apache.iotdb.commons.path.NonAlignedFullPath;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.storageengine.dataregion.read.filescan.IChunkHandle;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.filter.operator.TimeFilterOperators;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+@RunWith(Parameterized.class)
+public class WritableMemChunkRegionScanTest {
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {{0}, {1000}, {10000}, {20000}});
+ }
+
+ private int defaultTvListThreshold;
+ private int tvListSortThreshold;
+
+ public WritableMemChunkRegionScanTest(int tvListSortThreshold) {
+ this.tvListSortThreshold = tvListSortThreshold;
+ }
+
+ @Before
+ public void setup() {
+ defaultTvListThreshold = IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold();
+ IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(tvListSortThreshold);
+ }
+
+ @After
+ public void tearDown() {
+ IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(defaultTvListThreshold);
+ }
+
+ @Test
+ public void testAlignedWritableMemChunkRegionScan() {
+ PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
+ try {
+ List<IMeasurementSchema> measurementSchemas =
+ Arrays.asList(
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32),
+ new MeasurementSchema("s3", TSDataType.INT32));
+ AlignedWritableMemChunk writableMemChunk = null;
+ int size = 100000;
+ for (int i = 0; i < size; i++) {
+ if (i <= 10000) {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {1, null, 1});
+ } else if (i <= 20000) {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {null, null, 2});
+ } else if (i <= 30000) {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {3, null, null});
+ } else {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {4, 4, 4});
+ }
+ }
+ writableMemChunk =
+ (AlignedWritableMemChunk)
+ memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), "");
+ List<BitMap> bitMaps = new ArrayList<>();
+ long[] timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyList()),
+ bitMaps,
+ true,
+ null);
+ Assert.assertEquals(2, timestamps.length);
+ Assert.assertEquals(0, timestamps[0]);
+ Assert.assertFalse(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertFalse(bitMaps.get(0).isMarked(2));
+ Assert.assertTrue(bitMaps.get(1).isMarked(0));
+ Assert.assertFalse(bitMaps.get(1).isMarked(1));
+ Assert.assertTrue(bitMaps.get(1).isMarked(2));
+ Assert.assertEquals(30001, timestamps[1]);
+
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(new TimeRange(0, 12000))),
+ bitMaps,
+ true,
+ new TimeFilterOperators.TimeGt(10000000));
+ Assert.assertEquals(0, timestamps.length);
+
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(new TimeRange(0, 12000))),
+ bitMaps,
+ true,
+ new TimeFilterOperators.TimeGt(11000));
+
+ Assert.assertEquals(3, timestamps.length);
+ Assert.assertEquals(12001, timestamps[0]);
+ Assert.assertTrue(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertFalse(bitMaps.get(0).isMarked(2));
+ Assert.assertEquals(20001, timestamps[1]);
+ Assert.assertFalse(bitMaps.get(1).isMarked(0));
+ Assert.assertTrue(bitMaps.get(1).isMarked(1));
+ Assert.assertTrue(bitMaps.get(1).isMarked(2));
+ Assert.assertEquals(30001, timestamps[2]);
+ Assert.assertTrue(bitMaps.get(2).isMarked(0));
+ Assert.assertFalse(bitMaps.get(2).isMarked(1));
+ Assert.assertTrue(bitMaps.get(2).isMarked(2));
+
+ writableMemChunk.writeAlignedPoints(
+ 1000001, new Object[] {1, null, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(
+ 1000002, new Object[] {null, 1, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(1000002, new Object[] {1, 1, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(
+ 1000003, new Object[] {1, null, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(1000004, new Object[] {1, null, 1}, measurementSchemas);
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyList()),
+ bitMaps,
+ true,
+ new TimeFilterOperators.TimeGt(1000000));
+ Assert.assertEquals(3, timestamps.length);
+ Assert.assertEquals(1000001, timestamps[0]);
+ Assert.assertFalse(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertTrue(bitMaps.get(0).isMarked(2));
+ Assert.assertEquals(1000002, timestamps[1]);
+ Assert.assertTrue(bitMaps.get(1).isMarked(0));
+ Assert.assertFalse(bitMaps.get(1).isMarked(1));
+ Assert.assertTrue(bitMaps.get(1).isMarked(2));
+ Assert.assertEquals(1000004, timestamps[2]);
+ Assert.assertTrue(bitMaps.get(2).isMarked(0));
+ Assert.assertTrue(bitMaps.get(2).isMarked(1));
+ Assert.assertFalse(bitMaps.get(2).isMarked(2));
+
+ Map<String, List<IChunkHandle>> chunkHandleMap = new HashMap<>();
+ memTable.queryForDeviceRegionScan(
+ new StringArrayDeviceID("root.test.d1"),
+ true,
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1000000));
+ Assert.assertEquals(3, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {1000001, 1000001}, chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000002, 1000002}, chunkHandleMap.get("s2").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000004, 1000004}, chunkHandleMap.get("s3").get(0).getPageStatisticsTime());
+
+ memTable.queryForSeriesRegionScan(
+ new AlignedFullPath(
+ new StringArrayDeviceID("root.test.d1"),
+ IMeasurementSchema.getMeasurementNameList(measurementSchemas),
+ measurementSchemas),
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1000000));
+ Assert.assertEquals(3, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {1000001, 1000001}, chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000002, 1000002}, chunkHandleMap.get("s2").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000004, 1000004}, chunkHandleMap.get("s3").get(0).getPageStatisticsTime());
+ } finally {
+ memTable.release();
+ }
+ }
+
+ @Test
+ public void testTableWritableMemChunkRegionScan() {
+ List<IMeasurementSchema> measurementSchemas =
+ Arrays.asList(
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32),
+ new MeasurementSchema("s3", TSDataType.INT32));
+ AlignedWritableMemChunk writableMemChunk =
+ new AlignedWritableMemChunk(measurementSchemas, true);
+ int size = 100000;
+ for (int i = 0; i < size; i++) {
+ if (i <= 10000) {
+ writableMemChunk.writeAlignedPoints(i, new Object[] {1, null, 1}, measurementSchemas);
+ } else if (i <= 20000) {
+ writableMemChunk.writeAlignedPoints(i, new Object[] {null, null, 2}, measurementSchemas);
+ } else if (i <= 30000) {
+ writableMemChunk.writeAlignedPoints(i, new Object[] {3, null, null}, measurementSchemas);
+ } else {
+ writableMemChunk.writeAlignedPoints(i, new Object[] {4, 4, 4}, measurementSchemas);
+ }
+ }
+ List<BitMap> bitMaps = new ArrayList<>();
+ long[] timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(), Collections.emptyList(), Collections.emptyList()),
+ bitMaps,
+ false,
+ null);
+ Assert.assertEquals(1, timestamps.length);
+ Assert.assertEquals(0, timestamps[0]);
+ Assert.assertFalse(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertFalse(bitMaps.get(0).isMarked(2));
+
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(new TimeRange(0, 12000))),
+ bitMaps,
+ false,
+ new TimeFilterOperators.TimeGt(11000));
+
+ Assert.assertEquals(1, timestamps.length);
+ Assert.assertEquals(11001, timestamps[0]);
+ Assert.assertTrue(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertTrue(bitMaps.get(0).isMarked(2));
+ }
+
+ @Test
+ public void testNonAlignedWritableMemChunkRegionScan() {
+ PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
+ try {
+ MeasurementSchema measurementSchema = new MeasurementSchema("s1", TSDataType.INT32);
+ int size = 100000;
+ for (int i = 0; i < size; i++) {
+ memTable.write(
+ new StringArrayDeviceID("root.test.d1"),
+ Collections.singletonList(measurementSchema),
+ i,
+ new Object[] {i});
+ }
+ WritableMemChunk writableMemChunk =
+ (WritableMemChunk)
+ memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), "s1");
+ Optional<Long> timestamp = writableMemChunk.getAnySatisfiedTimestamp(null, null);
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(0, timestamp.get().longValue());
+
+ timestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ null, new TimeFilterOperators.TimeBetweenAnd(1000L, 2000L));
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(1000, timestamp.get().longValue());
+
+ timestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Collections.singletonList(new TimeRange(1, 1500)),
+ new TimeFilterOperators.TimeBetweenAnd(1000L, 2000L));
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(1501, timestamp.get().longValue());
+
+ timestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Collections.singletonList(new TimeRange(1, 1500)),
+ new TimeFilterOperators.TimeBetweenAnd(100000L, 200000L));
+ Assert.assertFalse(timestamp.isPresent());
+
+ Map<String, List<IChunkHandle>> chunkHandleMap = new HashMap<>();
+ memTable.queryForDeviceRegionScan(
+ new StringArrayDeviceID("root.test.d1"),
+ false,
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1));
+ Assert.assertEquals(1, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {2, 2}, chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ memTable.queryForSeriesRegionScan(
+ new NonAlignedFullPath(new StringArrayDeviceID("root.test.d1"), measurementSchema),
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1));
+ Assert.assertEquals(1, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {2, 2}, chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ } finally {
+ memTable.release();
+ }
+ }
+}
(iotdb) 04/05: Bump at.yawk.lz4:lz4-java from 1.10.0 to 1.10.1 (#16874)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 30d6845c2b819cc86198910bd15f71f8fde2203e
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Sat Dec 6 20:55:37 2025 +0800
Bump at.yawk.lz4:lz4-java from 1.10.0 to 1.10.1 (#16874)
Bumps [at.yawk.lz4:lz4-java](https://github.com/yawkat/lz4-java) from 1.10.0 to 1.10.1.
- [Release notes](https://github.com/yawkat/lz4-java/releases)
- [Changelog](https://github.com/yawkat/lz4-java/blob/main/CHANGES.md)
- [Commits](https://github.com/yawkat/lz4-java/compare/v1.10.0...v1.10.1)
---
updated-dependencies:
- dependency-name: at.yawk.lz4:lz4-java
dependency-version: 1.10.1
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
(cherry picked from commit 0bb9e96ece7f1bc2b08541f773ea346186e66e01)
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 7b26e1dcd0e..13d51e6d812 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,7 +118,7 @@
<junit.version>4.13.2</junit.version>
<!-- This was the last version to support Java 8 -->
<logback.version>1.3.16</logback.version>
- <lz4-java.version>1.10.0</lz4-java.version>
+ <lz4-java.version>1.10.1</lz4-java.version>
<maven.assembly.version>3.6.0</maven.assembly.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>