You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/10/28 09:41:46 UTC
(iotdb) branch ty/AggPerf created (now 4c31d460418)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/AggPerf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 4c31d460418 make some opt
This branch includes the following new commits:
new 7a656e937d2 Add write example
new ad3a90bcbe8 Add write for aggTest
new 4688193eb0a change print unit
new 239b8459928 Merge remote-tracking branch 'origin/master' into ty/AggPerf
new 4c31d460418 make some opt
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 04/05: Merge remote-tracking branch 'origin/master' into ty/AggPerf
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/AggPerf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 239b845992814c63fff9499ac0411cad17aec516
Merge: 4688193eb0a db62bf92cea
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Oct 28 16:39:12 2024 +0800
Merge remote-tracking branch 'origin/master' into ty/AggPerf
.../fetcher/TableDeviceSchemaValidator.java | 62 +++++++++++++++-------
.../GeneralRegionAttributeSecurityService.java | 4 +-
.../db/schemaengine/table/DataNodeTableCache.java | 6 +++
.../apache/iotdb/commons/utils/StatusUtils.java | 1 +
4 files changed, 51 insertions(+), 22 deletions(-)
(iotdb) 01/05: Add write example
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/AggPerf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7a656e937d2ec2dea89c2f98ca1d640ec0271d1a
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Oct 28 14:28:02 2024 +0800
Add write example
---
.../src/main/java/org/apache/iotdb/TableTest.java | 214 +++++++++++++++++++++
1 file changed, 214 insertions(+)
diff --git a/example/session/src/main/java/org/apache/iotdb/TableTest.java b/example/session/src/main/java/org/apache/iotdb/TableTest.java
new file mode 100644
index 00000000000..b90782d5ff6
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/TableTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.isession.IPooledSession;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.pool.SessionPool;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TableTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TableTest.class);
+
+ private static final String TABLE_SQL_DIALECT = "table";
+
+ private static final AtomicInteger deviceIdGenerator = new AtomicInteger(0);
+
+ private static final List<IMeasurementSchema> TABLE_SCHEMA_LIST = new ArrayList<>();
+ private static final List<Tablet.ColumnType> TABLE_COLUMN_TYPES =
+ Arrays.asList(
+ Tablet.ColumnType.ID,
+ Tablet.ColumnType.ID,
+ Tablet.ColumnType.ID,
+ Tablet.ColumnType.ATTRIBUTE,
+ Tablet.ColumnType.MEASUREMENT,
+ Tablet.ColumnType.MEASUREMENT,
+ Tablet.ColumnType.MEASUREMENT);
+
+ private static final String COLUMN_NAME_1 = "city";
+ private static final String COLUMN_NAME_2 = "region";
+ private static final String COLUMN_NAME_3 = "device_id";
+ private static final String COLUMN_NAME_4 = "color";
+ private static final String COLUMN_NAME_5 = "s1";
+ private static final String COLUMN_NAME_6 = "s2";
+ private static final String COLUMN_NAME_7 = "s3";
+
+ private static final List<IMeasurementSchema> TREE_SCHEMA_LIST = new ArrayList<>();
+
+ static {
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_1, TSDataType.STRING));
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_2, TSDataType.STRING));
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_3, TSDataType.STRING));
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_4, TSDataType.STRING));
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_5, TSDataType.DOUBLE));
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_6, TSDataType.DOUBLE));
+ TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_7, TSDataType.DOUBLE));
+
+ TREE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_5, TSDataType.DOUBLE));
+ TREE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_6, TSDataType.DOUBLE));
+ TREE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_7, TSDataType.DOUBLE));
+ }
+
+ // 2024-10-01T00:00:00+08:00
+ private static final long START_TIME = 1727712000000L;
+
+ public static void main(String[] args) {
+
+ // table
+ String sqlDialect = args[0];
+ // 127.0.0.1
+ String ip = args[1];
+ // 10
+ int maxSize = Integer.parseInt(args[2]);
+
+ String database = args[3];
+
+ int deviceNum = Integer.parseInt(args[4]);
+
+ SessionPool sessionPool =
+ new SessionPool.Builder()
+ .host(ip)
+ .port(6667)
+ .user("root")
+ .password("root")
+ .maxSize(maxSize)
+ .sqlDialect(sqlDialect)
+ .database(database)
+ .build();
+
+ if (TABLE_SQL_DIALECT.equalsIgnoreCase(sqlDialect)) {
+ for (int i = 0; i < maxSize; i++) {
+ new Thread(() -> writeTable(sessionPool, deviceNum)).start();
+ }
+ } else {
+ for (int i = 0; i < maxSize; i++) {
+ new Thread(() -> writeTree(sessionPool, deviceNum, database)).start();
+ }
+ }
+ }
+
+ private static void writeTable(final SessionPool sessionPool, final int deviceNum) {
+
+ try (IPooledSession session = sessionPool.getPooledSession()) {
+ while (true) {
+ int device = deviceIdGenerator.getAndIncrement();
+ if (device >= deviceNum) {
+ break;
+ }
+ int city = device % 10;
+ int region = device % 100;
+ int color = device % 5;
+
+ Tablet tablet = new Tablet("table1", TABLE_SCHEMA_LIST, TABLE_COLUMN_TYPES, 10000);
+ String cityId = "city_" + city;
+ String regionId = "region_" + region;
+ String deviceId = "d_" + device;
+ String colorId = "color_" + color;
+ for (int i = 0; i < 60 * 60 * 24 * 30; i++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, START_TIME + i);
+ tablet.addValue(COLUMN_NAME_1, rowIndex, cityId);
+ tablet.addValue(COLUMN_NAME_2, rowIndex, regionId);
+ tablet.addValue(COLUMN_NAME_3, rowIndex, deviceId);
+ tablet.addValue(COLUMN_NAME_4, rowIndex, colorId);
+ tablet.addValue(COLUMN_NAME_5, rowIndex, i * 1.0d);
+ tablet.addValue(COLUMN_NAME_6, rowIndex, i * 1.0d);
+ tablet.addValue(COLUMN_NAME_7, rowIndex, i * 1.0d);
+
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ session.insertTablet(tablet);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertTablet(tablet);
+ tablet.reset();
+ }
+ }
+
+ } catch (IoTDBConnectionException e) {
+ LOGGER.error("Connection error", e);
+ throw new RuntimeException(e);
+ } catch (StatementExecutionException e) {
+ LOGGER.error("Execute error", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void writeTree(
+ final SessionPool sessionPool, final int deviceNum, final String database) {
+
+ try {
+ while (true) {
+ int device = deviceIdGenerator.getAndIncrement();
+ if (device >= deviceNum) {
+ break;
+ }
+ int city = device % 10;
+ int region = device % 100;
+ // int color = device % 5;
+
+ Tablet tablet =
+ new Tablet(
+ String.format(
+ "%s.%s.%s.%s", database, "city_" + city, "region_" + region, "d_" + device),
+ TREE_SCHEMA_LIST,
+ 10000);
+ for (int i = 0; i < 60 * 60 * 24 * 30; i++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, START_TIME + i);
+ tablet.addValue(COLUMN_NAME_5, rowIndex, i * 1.0d);
+ tablet.addValue(COLUMN_NAME_6, rowIndex, i * 1.0d);
+ tablet.addValue(COLUMN_NAME_7, rowIndex, i * 1.0d);
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ sessionPool.insertTablet(tablet);
+ tablet.reset();
+ }
+ }
+
+ if (tablet.rowSize != 0) {
+ sessionPool.insertTablet(tablet);
+ tablet.reset();
+ }
+ }
+
+ } catch (IoTDBConnectionException e) {
+ LOGGER.error("Connection error", e);
+ throw new RuntimeException(e);
+ } catch (StatementExecutionException e) {
+ LOGGER.error("Execute error", e);
+ throw new RuntimeException(e);
+ }
+ }
+}
(iotdb) 02/05: Add write for aggTest
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/AggPerf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ad3a90bcbe8ac9c6a6d948b62e711485fef3923b
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Oct 28 16:15:22 2024 +0800
Add write for aggTest
---
.../src/main/java/org/apache/iotdb/TableTest.java | 92 ++++++++++++++++++----
1 file changed, 75 insertions(+), 17 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/TableTest.java b/example/session/src/main/java/org/apache/iotdb/TableTest.java
index b90782d5ff6..3a592f6060f 100644
--- a/example/session/src/main/java/org/apache/iotdb/TableTest.java
+++ b/example/session/src/main/java/org/apache/iotdb/TableTest.java
@@ -50,7 +50,7 @@ public class TableTest {
Tablet.ColumnType.ID,
Tablet.ColumnType.ID,
Tablet.ColumnType.ID,
- Tablet.ColumnType.ATTRIBUTE,
+ // Tablet.ColumnType.ATTRIBUTE,
Tablet.ColumnType.MEASUREMENT,
Tablet.ColumnType.MEASUREMENT,
Tablet.ColumnType.MEASUREMENT);
@@ -69,7 +69,7 @@ public class TableTest {
TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_1, TSDataType.STRING));
TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_2, TSDataType.STRING));
TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_3, TSDataType.STRING));
- TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_4, TSDataType.STRING));
+ // TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_4, TSDataType.STRING));
TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_5, TSDataType.DOUBLE));
TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_6, TSDataType.DOUBLE));
TABLE_SCHEMA_LIST.add(new MeasurementSchema(COLUMN_NAME_7, TSDataType.DOUBLE));
@@ -93,7 +93,7 @@ public class TableTest {
String database = args[3];
- int deviceNum = Integer.parseInt(args[4]);
+ final int deviceNum = Integer.parseInt(args[4]);
SessionPool sessionPool =
new SessionPool.Builder()
@@ -106,21 +106,40 @@ public class TableTest {
.database(database)
.build();
+ long startTime = System.nanoTime();
+ List<Thread> subThreads = new ArrayList<>(maxSize);
if (TABLE_SQL_DIALECT.equalsIgnoreCase(sqlDialect)) {
+ // CREATE TABLE table1(city STRING ID, region STRING ID, device_id STRING ID, color STRING
+ // ATTRIBUTE, s1 DOUBLE MEASUREMENT, s2 DOUBLE MEASUREMENT, s3 DOUBLE MEASUREMENT)
for (int i = 0; i < maxSize; i++) {
- new Thread(() -> writeTable(sessionPool, deviceNum)).start();
+ Thread t = new Thread(() -> writeTable(sessionPool, deviceNum));
+ subThreads.add(t);
+ t.start();
}
} else {
for (int i = 0; i < maxSize; i++) {
- new Thread(() -> writeTree(sessionPool, deviceNum, database)).start();
+ Thread t = new Thread(() -> writeTree(sessionPool, deviceNum, database));
+ subThreads.add(t);
+ t.start();
}
}
+ subThreads.forEach(
+ t -> {
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ LOGGER.info(
+ "{} writing {} devices costs {}ms", sqlDialect, deviceNum, System.nanoTime() - startTime);
}
private static void writeTable(final SessionPool sessionPool, final int deviceNum) {
try (IPooledSession session = sessionPool.getPooledSession()) {
while (true) {
+ long startTime = System.nanoTime();
int device = deviceIdGenerator.getAndIncrement();
if (device >= deviceNum) {
break;
@@ -129,18 +148,19 @@ public class TableTest {
int region = device % 100;
int color = device % 5;
- Tablet tablet = new Tablet("table1", TABLE_SCHEMA_LIST, TABLE_COLUMN_TYPES, 10000);
+ Tablet tablet = new Tablet("table1", TABLE_SCHEMA_LIST, TABLE_COLUMN_TYPES, 10_000);
String cityId = "city_" + city;
String regionId = "region_" + region;
String deviceId = "d_" + device;
String colorId = "color_" + color;
- for (int i = 0; i < 60 * 60 * 24 * 30; i++) {
+ long roundStartTime = System.nanoTime();
+ for (int i = 0; i < 6 * 60 * 24 * 30; i++) {
int rowIndex = tablet.rowSize++;
- tablet.addTimestamp(rowIndex, START_TIME + i);
+ tablet.addTimestamp(rowIndex, START_TIME + i * 10_000L);
tablet.addValue(COLUMN_NAME_1, rowIndex, cityId);
tablet.addValue(COLUMN_NAME_2, rowIndex, regionId);
tablet.addValue(COLUMN_NAME_3, rowIndex, deviceId);
- tablet.addValue(COLUMN_NAME_4, rowIndex, colorId);
+ // tablet.addValue(COLUMN_NAME_4, rowIndex, colorId);
tablet.addValue(COLUMN_NAME_5, rowIndex, i * 1.0d);
tablet.addValue(COLUMN_NAME_6, rowIndex, i * 1.0d);
tablet.addValue(COLUMN_NAME_7, rowIndex, i * 1.0d);
@@ -148,6 +168,15 @@ public class TableTest {
if (tablet.rowSize == tablet.getMaxRowNumber()) {
session.insertTablet(tablet);
tablet.reset();
+ long writtenRows = i + 1L;
+ if (writtenRows % 100_000 == 0) {
+ LOGGER.info(
+ "Device {} has written {} rows, time cost is {}ms",
+ deviceId,
+ writtenRows,
+ (System.nanoTime() - roundStartTime) / 1_000_000);
+ roundStartTime = System.nanoTime();
+ }
}
}
@@ -155,6 +184,21 @@ public class TableTest {
session.insertTablet(tablet);
tablet.reset();
}
+ session.executeNonQueryStatement(
+ String.format(
+ "UPDATE table1 SET %s='%s' WHERE %s='%s' AND %s='%s' AND %s='%s'",
+ COLUMN_NAME_4,
+ colorId,
+ COLUMN_NAME_1,
+ cityId,
+ COLUMN_NAME_2,
+ regionId,
+ COLUMN_NAME_3,
+ deviceId));
+ LOGGER.info(
+ "Device {} finished, total time cost is {}ms",
+ deviceId,
+ (System.nanoTime() - startTime) / 1_000_000);
}
} catch (IoTDBConnectionException e) {
@@ -171,6 +215,7 @@ public class TableTest {
try {
while (true) {
+ long startTime = System.nanoTime();
int device = deviceIdGenerator.getAndIncrement();
if (device >= deviceNum) {
break;
@@ -179,21 +224,29 @@ public class TableTest {
int region = device % 100;
// int color = device % 5;
- Tablet tablet =
- new Tablet(
- String.format(
- "%s.%s.%s.%s", database, "city_" + city, "region_" + region, "d_" + device),
- TREE_SCHEMA_LIST,
- 10000);
- for (int i = 0; i < 60 * 60 * 24 * 30; i++) {
+ String deviceId =
+ String.format(
+ "%s.%s.%s.%s", database, "city_" + city, "region_" + region, "d_" + device);
+ Tablet tablet = new Tablet(deviceId, TREE_SCHEMA_LIST, 10000);
+ long roundStartTime = System.nanoTime();
+ for (int i = 0; i < 6 * 60 * 24 * 30; i++) {
int rowIndex = tablet.rowSize++;
- tablet.addTimestamp(rowIndex, START_TIME + i);
+ tablet.addTimestamp(rowIndex, START_TIME + i * 10_000L);
tablet.addValue(COLUMN_NAME_5, rowIndex, i * 1.0d);
tablet.addValue(COLUMN_NAME_6, rowIndex, i * 1.0d);
tablet.addValue(COLUMN_NAME_7, rowIndex, i * 1.0d);
if (tablet.rowSize == tablet.getMaxRowNumber()) {
sessionPool.insertTablet(tablet);
tablet.reset();
+ long writtenRows = i + 1L;
+ if (writtenRows % 100_000 == 0) {
+ LOGGER.info(
+ "Device {} has written {} rows, time cost is {}ms",
+ deviceId,
+ writtenRows,
+ (System.nanoTime() - roundStartTime) / 1_000_000);
+ roundStartTime = System.nanoTime();
+ }
}
}
@@ -201,6 +254,11 @@ public class TableTest {
sessionPool.insertTablet(tablet);
tablet.reset();
}
+
+ LOGGER.info(
+ "Device {} finished, total time cost is {}ms",
+ deviceId,
+ (System.nanoTime() - startTime) / 1_000_000);
}
} catch (IoTDBConnectionException e) {
(iotdb) 05/05: make some opt
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/AggPerf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4c31d460418b7873270199759e187be0dfab237e
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Oct 28 17:41:28 2024 +0800
make some opt
---
.../execution/operator/source/SeriesScanUtil.java | 19 +++++++++++--------
.../plan/planner/TableOperatorGenerator.java | 2 +-
.../planner/plan/parameter/SeriesScanOptions.java | 11 ++++++++++-
.../db/storageengine/dataregion/DataRegion.java | 2 +-
.../dataregion/read/QueryDataSource.java | 20 +++++++++++++++++---
5 files changed, 40 insertions(+), 14 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
index 1389666f0b2..15fe07161a6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
@@ -183,15 +183,18 @@ public class SeriesScanUtil implements Accountable {
// differentiate the data of tree model and table model.
if (context.isIgnoreAllNullRows()) {
ttl = DataNodeTTLCache.getInstance().getTTLForTree(deviceID);
+ scanOptions.setTTL(ttl);
} else {
- String databaseName = dataSource.getDatabaseName();
- ttl =
- databaseName == null
- ? Long.MAX_VALUE
- : DataNodeTTLCache.getInstance()
- .getTTLForTable(databaseName, deviceID.getTableName());
- }
- scanOptions.setTTL(ttl);
+ if (scanOptions.timeFilterNeedUpdatedByTll()) {
+ String databaseName = dataSource.getDatabaseName();
+ ttl =
+ databaseName == null
+ ? Long.MAX_VALUE
+ : DataNodeTTLCache.getInstance()
+ .getTTLForTable(databaseName, deviceID.getTableName());
+ scanOptions.setTTL(ttl);
+ }
+ }
// init file index
orderUtils.setCurSeqFileIndex(dataSource);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index 56d6868a24b..d3150c2f731 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -1427,7 +1427,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
groupByChannels,
aggregatorBuilder.build(),
node.getStep(),
- 10_000,
+ 64,
Long.MAX_VALUE,
false,
Long.MAX_VALUE);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java
index dea62b9cf05..25853c9c547 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java
@@ -32,11 +32,14 @@ import org.apache.tsfile.read.reader.series.PaginationController;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
public class SeriesScanOptions {
private Filter globalTimeFilter;
+ private final AtomicBoolean timeFilterUpdatedByTll = new AtomicBoolean(false);
+
private final Filter pushDownFilter;
private final long pushDownLimit;
@@ -97,8 +100,14 @@ public class SeriesScanOptions {
}
}
+ public boolean timeFilterNeedUpdatedByTll() {
+ return !timeFilterUpdatedByTll.get();
+ }
+
public void setTTL(long dataTTL) {
- this.globalTimeFilter = updateFilterUsingTTL(globalTimeFilter, dataTTL);
+ if (timeFilterUpdatedByTll.compareAndSet(false, true)) {
+ this.globalTimeFilter = updateFilterUsingTTL(globalTimeFilter, dataTTL);
+ }
}
/**
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 16654ea8b16..e05ca6222eb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2012,7 +2012,7 @@ public class DataRegion implements IDataRegionForQuery {
QUERY_RESOURCE_METRIC_SET.recordQueryResourceNum(SEQUENCE_TSFILE, seqResources.size());
QUERY_RESOURCE_METRIC_SET.recordQueryResourceNum(UNSEQUENCE_TSFILE, unseqResources.size());
- return new QueryDataSource(seqResources, unseqResources);
+ return new QueryDataSource(seqResources, unseqResources, databaseName);
} catch (MetadataException e) {
throw new QueryProcessException(e);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java
index dc56ad11a12..4748cc20e58 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java
@@ -64,6 +64,8 @@ public class QueryDataSource implements IQueryDataSource {
/* The traversal order of unseqResources (different for each device) */
private int[] unSeqFileOrderIndex;
+ private String databaseName = null;
+
private static final Comparator<Long> descendingComparator = (o1, o2) -> Long.compare(o2, o1);
public QueryDataSource(List<TsFileResource> seqResources, List<TsFileResource> unseqResources) {
@@ -71,12 +73,20 @@ public class QueryDataSource implements IQueryDataSource {
this.unseqResources = unseqResources;
}
+ public QueryDataSource(
+ List<TsFileResource> seqResources, List<TsFileResource> unseqResources, String databaseName) {
+ this.seqResources = seqResources;
+ this.unseqResources = unseqResources;
+ this.databaseName = databaseName;
+ }
+
// used for compaction, because in compaction task(unlike query, each QueryDataSource only serve
// for one series), we will reuse this object for multi series
public QueryDataSource(QueryDataSource other) {
this.seqResources = other.seqResources;
this.unseqResources = other.unseqResources;
this.unSeqFileOrderIndex = other.unSeqFileOrderIndex;
+ this.databaseName = other.databaseName;
}
public List<TsFileResource> getSeqResources() {
@@ -89,7 +99,8 @@ public class QueryDataSource implements IQueryDataSource {
@Override
public IQueryDataSource clone() {
- QueryDataSource queryDataSource = new QueryDataSource(getSeqResources(), getUnseqResources());
+ QueryDataSource queryDataSource =
+ new QueryDataSource(getSeqResources(), getUnseqResources(), databaseName);
queryDataSource.setSingleDevice(isSingleDevice());
return queryDataSource;
}
@@ -226,7 +237,10 @@ public class QueryDataSource implements IQueryDataSource {
}
public String getDatabaseName() {
- List<TsFileResource> resources = !seqResources.isEmpty() ? seqResources : unseqResources;
- return resources.isEmpty() ? null : resources.get(0).getDatabaseName();
+ if (databaseName == null) {
+ List<TsFileResource> resources = !seqResources.isEmpty() ? seqResources : unseqResources;
+ databaseName = resources.isEmpty() ? null : resources.get(0).getDatabaseName();
+ }
+ return databaseName;
}
}
(iotdb) 03/05: change print unit
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/AggPerf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4688193eb0ade52df91fd0a9be2c9b0bb9ff1dc0
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Oct 28 16:28:18 2024 +0800
change print unit
---
example/session/pom.xml | 27 ++++++++++++++++++++++
.../src/main/java/org/apache/iotdb/TableTest.java | 5 +++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/example/session/pom.xml b/example/session/pom.xml
index a7bdd7aa124..ef956773723 100644
--- a/example/session/pom.xml
+++ b/example/session/pom.xml
@@ -40,4 +40,31 @@
<version>${project.version}</version>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.3.0</version>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.iotdb.TableTest</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/example/session/src/main/java/org/apache/iotdb/TableTest.java b/example/session/src/main/java/org/apache/iotdb/TableTest.java
index 3a592f6060f..ceabb8db1d2 100644
--- a/example/session/src/main/java/org/apache/iotdb/TableTest.java
+++ b/example/session/src/main/java/org/apache/iotdb/TableTest.java
@@ -132,7 +132,10 @@ public class TableTest {
}
});
LOGGER.info(
- "{} writing {} devices costs {}ms", sqlDialect, deviceNum, System.nanoTime() - startTime);
+ "{} writing {} devices costs {}s",
+ sqlDialect,
+ deviceNum,
+ (System.nanoTime() - startTime) / 1_000_000_000L);
}
private static void writeTable(final SessionPool sessionPool, final int deviceNum) {