You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/06/22 10:48:43 UTC
[iotdb] branch QueryPerformace updated (b76b253b51 -> 8395f1b967)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch QueryPerformace
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from b76b253b51 change last query
add 8ecdbb15ad [IOTDB-3494] Fix Python-Client test (#6362)
add 2f486bfc90 [IOTDB-3585] Support set/getFetchSize() in IT framework (#6364)
add 7dde745854 Update the README of the integration-test module (#6367)
add 2f0218feee [IOTDB-3522] Set heartbeat as a periodic scheduled task (#6328)
add 1c2783cfe8 [IOTDB-3582] Fix client connections leak caused by delete storage group (#6360)
add 05f02a63e1 [IOTDB-3536] ConfigNode : org.apache.thrift.TException: Error in calling method flush (#6340)
add 4f3890471c fix the issue that ALTER timeseries is not correct in NewIoTDB (#6365)
add cf31069c39 accelerate countAccummulator
add bcfbea543d use binary search in skipOutOfTimeRangePoints
add 8145006af6 Fix bugs
add 1430c63e85 return rowIndex in processTsBLock
add b57604cbe6 [IOTDB-3545] Add permission and SQL relation table in doc (#6369)
add fbb74cd83e Add comment on getLeader (#6371)
add f65514eea5 [IOTDB-3584] Snapshot unstable due to segment size (#6370)
add ff5a14a86d [IOTDB-3577] Add semantic checks for linear fill (#6372)
add 4a01899bc1 [IOTDB-3573] Fix IT test stucked by JDBC infinite network timeout (#6361)
add 37cd367478 Merge branch 'master' into accelerateCount
add 438abe0c78 Fix IT bugs
new 960fe9e0b0 Merge remote-tracking branch 'origin/accelerateCount' into QueryPerformace
new 8395f1b967 test
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/standalone-it-for-mpp.yml | 2 +-
client-py/tests/test_dataframe.py | 2 +
client-py/tests/test_one_device.py | 1 +
client-py/tests/test_tablet.py | 2 +
client-py/tests/test_todf.py | 3 +
.../resources/conf/iotdb-confignode.properties | 4 -
.../confignode/client/AsyncDataNodeClientPool.java | 18 +
.../confignode/client/SyncDataNodeClientPool.java | 21 +-
...ionManagementHandler.java => FlushHandler.java} | 28 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 11 -
.../confignode/conf/ConfigNodeDescriptor.java | 5 -
.../statemachine/PartitionRegionStateMachine.java | 2 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/Manager.java | 3 +
.../iotdb/confignode/manager/NodeManager.java | 27 +
.../iotdb/confignode/manager/UDFManager.java | 18 +-
.../iotdb/confignode/manager/load/LoadManager.java | 76 ++-
.../thrift/ConfigNodeRPCServiceProcessor.java | 15 +-
.../apache/iotdb/consensus/config/RatisConfig.java | 4 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 9 +
.../Administration-Management/Administration.md | 288 ++++++++-
.../Administration-Management/Administration.md | 287 ++++++++-
.../main/java/org/apache/iotdb/SessionExample.java | 708 +--------------------
integration-test/README.md | 95 +--
integration-test/pom.xml | 466 +++++---------
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 31 +-
.../apache/iotdb/it/env/AbstractNodeWrapper.java | 18 +-
.../iotdb/itbase/runtime/ClusterTestStatement.java | 15 +-
.../main/java/org/apache/iotdb/jdbc/Config.java | 3 +
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 11 +-
.../apache/iotdb/jdbc/IoTDBConnectionParams.java | 13 +
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 4 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +-
.../iotdb/db/localconfignode/LocalConfigNode.java | 2 +-
.../iotdb/db/mpp/aggregation/Accumulator.java | 8 +-
.../iotdb/db/mpp/aggregation/Aggregator.java | 9 +-
.../iotdb/db/mpp/aggregation/AvgAccumulator.java | 34 +-
.../iotdb/db/mpp/aggregation/CountAccumulator.java | 24 +-
.../db/mpp/aggregation/ExtremeAccumulator.java | 34 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 80 ++-
.../mpp/aggregation/FirstValueDescAccumulator.java | 48 +-
.../db/mpp/aggregation/LastValueAccumulator.java | 68 +-
.../mpp/aggregation/LastValueDescAccumulator.java | 60 +-
.../db/mpp/aggregation/MaxTimeAccumulator.java | 8 +-
.../db/mpp/aggregation/MaxTimeDescAccumulator.java | 10 +-
.../db/mpp/aggregation/MaxValueAccumulator.java | 34 +-
.../db/mpp/aggregation/MinTimeAccumulator.java | 10 +-
.../db/mpp/aggregation/MinTimeDescAccumulator.java | 8 +-
.../db/mpp/aggregation/MinValueAccumulator.java | 34 +-
.../iotdb/db/mpp/aggregation/SumAccumulator.java | 34 +-
.../slidingwindow/SlidingWindowAggregator.java | 3 +-
.../process/RawDataAggregationOperator.java | 45 +-
.../process/SlidingWindowAggregationOperator.java | 14 +-
.../AlignedSeriesAggregationScanOperator.java | 37 +-
.../source/SeriesAggregationScanOperator.java | 37 +-
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 32 +-
...piServiceIT.java => GrafanaApiServiceTest.java} | 2 +-
...estServiceIT.java => IoTDBRestServiceTest.java} | 2 +-
.../org/apache/iotdb/db/script/EnvScriptIT.java | 1 +
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 12 +
.../read/common/block/column/TimeColumn.java | 4 +
61 files changed, 1389 insertions(+), 1517 deletions(-)
copy confignode/src/main/java/org/apache/iotdb/confignode/client/handlers/{FunctionManagementHandler.java => FlushHandler.java} (65%)
rename server/src/test/java/org/apache/iotdb/db/protocol/rest/{GrafanaApiServiceIT.java => GrafanaApiServiceTest.java} (99%)
rename server/src/test/java/org/apache/iotdb/db/protocol/rest/{IoTDBRestServiceIT.java => IoTDBRestServiceTest.java} (99%)
[iotdb] 01/02: Merge remote-tracking branch 'origin/accelerateCount' into QueryPerformace
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryPerformace
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 960fe9e0b0996056949d30657cb6035eefe40812
Merge: b76b253b51 438abe0c78
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Jun 22 14:49:23 2022 +0800
Merge remote-tracking branch 'origin/accelerateCount' into QueryPerformace
.github/workflows/standalone-it-for-mpp.yml | 2 +-
client-py/tests/test_dataframe.py | 2 +
client-py/tests/test_one_device.py | 1 +
client-py/tests/test_tablet.py | 2 +
client-py/tests/test_todf.py | 3 +
.../resources/conf/iotdb-confignode.properties | 4 -
.../confignode/client/AsyncDataNodeClientPool.java | 18 +
.../confignode/client/SyncDataNodeClientPool.java | 21 +-
.../confignode/client/handlers/FlushHandler.java | 66 +++
.../iotdb/confignode/conf/ConfigNodeConfig.java | 11 -
.../confignode/conf/ConfigNodeDescriptor.java | 5 -
.../statemachine/PartitionRegionStateMachine.java | 2 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/Manager.java | 3 +
.../iotdb/confignode/manager/NodeManager.java | 27 ++
.../iotdb/confignode/manager/UDFManager.java | 18 +-
.../iotdb/confignode/manager/load/LoadManager.java | 76 ++--
.../thrift/ConfigNodeRPCServiceProcessor.java | 15 +-
.../apache/iotdb/consensus/config/RatisConfig.java | 4 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 9 +
.../Administration-Management/Administration.md | 288 +++++++++++--
.../Administration-Management/Administration.md | 287 +++++++++++--
integration-test/README.md | 95 +----
integration-test/pom.xml | 466 +++++++--------------
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 31 +-
.../apache/iotdb/it/env/AbstractNodeWrapper.java | 18 +-
.../iotdb/itbase/runtime/ClusterTestStatement.java | 15 +-
.../main/java/org/apache/iotdb/jdbc/Config.java | 3 +
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 11 +-
.../apache/iotdb/jdbc/IoTDBConnectionParams.java | 13 +
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 4 +
.../iotdb/db/localconfignode/LocalConfigNode.java | 2 +-
.../iotdb/db/mpp/aggregation/Accumulator.java | 8 +-
.../iotdb/db/mpp/aggregation/Aggregator.java | 9 +-
.../iotdb/db/mpp/aggregation/AvgAccumulator.java | 34 +-
.../iotdb/db/mpp/aggregation/CountAccumulator.java | 24 +-
.../db/mpp/aggregation/ExtremeAccumulator.java | 34 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 80 ++--
.../mpp/aggregation/FirstValueDescAccumulator.java | 48 ++-
.../db/mpp/aggregation/LastValueAccumulator.java | 68 +--
.../mpp/aggregation/LastValueDescAccumulator.java | 60 ++-
.../db/mpp/aggregation/MaxTimeAccumulator.java | 8 +-
.../db/mpp/aggregation/MaxTimeDescAccumulator.java | 10 +-
.../db/mpp/aggregation/MaxValueAccumulator.java | 34 +-
.../db/mpp/aggregation/MinTimeAccumulator.java | 10 +-
.../db/mpp/aggregation/MinTimeDescAccumulator.java | 8 +-
.../db/mpp/aggregation/MinValueAccumulator.java | 34 +-
.../iotdb/db/mpp/aggregation/SumAccumulator.java | 34 +-
.../slidingwindow/SlidingWindowAggregator.java | 3 +-
.../process/RawDataAggregationOperator.java | 45 +-
.../process/SlidingWindowAggregationOperator.java | 14 +-
.../AlignedSeriesAggregationScanOperator.java | 37 +-
.../source/SeriesAggregationScanOperator.java | 37 +-
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 32 +-
...piServiceIT.java => GrafanaApiServiceTest.java} | 2 +-
...estServiceIT.java => IoTDBRestServiceTest.java} | 2 +-
.../org/apache/iotdb/db/script/EnvScriptIT.java | 1 +
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 12 +
.../read/common/block/column/TimeColumn.java | 4 +
59 files changed, 1401 insertions(+), 829 deletions(-)
[iotdb] 02/02: test
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryPerformace
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8395f1b9677492109049b81d11b0f7b417a1b41f
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Jun 22 14:52:32 2022 +0800
test
---
.../main/java/org/apache/iotdb/SessionExample.java | 708 +--------------------
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +-
2 files changed, 36 insertions(+), 678 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index b86a4cbcf7..10c177c1ac 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -21,37 +21,23 @@ package org.apache.iotdb;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.session.Session;
-import org.apache.iotdb.session.SessionDataSet;
-import org.apache.iotdb.session.SessionDataSet.DataIterator;
-import org.apache.iotdb.session.template.MeasurementNode;
-import org.apache.iotdb.session.template.Template;
import org.apache.iotdb.session.util.Version;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.write.record.Tablet;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Random;
@SuppressWarnings("squid:S106")
public class SessionExample {
private static Session session;
- private static Session sessionEnableRedirect;
private static final String ROOT_SG1_D1_S1 = "root.sg1.d1.s1";
- private static final String ROOT_SG1_D1_S2 = "root.sg1.d1.s2";
- private static final String ROOT_SG1_D1_S3 = "root.sg1.d1.s3";
- private static final String ROOT_SG1_D1_S4 = "root.sg1.d1.s4";
- private static final String ROOT_SG1_D1_S5 = "root.sg1.d1.s5";
private static final String ROOT_SG1_D1 = "root.sg1.d1";
private static final String LOCAL_HOST = "127.0.0.1";
@@ -70,67 +56,14 @@ public class SessionExample {
// set session fetchSize
session.setFetchSize(10000);
- try {
- session.setStorageGroup("root.sg1");
- } catch (StatementExecutionException e) {
- if (e.getStatusCode() != TSStatusCode.PATH_ALREADY_EXIST_ERROR.getStatusCode()) {
- throw e;
- }
- }
-
- // createTemplate();
- createTimeseries();
- createMultiTimeseries();
- insertRecord();
- insertTablet();
- // insertTabletWithNullValues();
- // insertTablets();
- // insertRecords();
- // selectInto();
- // createAndDropContinuousQueries();
- // nonQuery();
- // query();
- // queryWithTimeout();
- // rawDataQuery();
- // lastDataQuery();
- // queryByIterator();
- // deleteData();
- // deleteTimeseries();
- // setTimeout();
+ // createTimeseries();
- sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root");
- sessionEnableRedirect.setEnableQueryRedirection(true);
- sessionEnableRedirect.open(false);
+ // insertTablet();
- // set session fetchSize
- sessionEnableRedirect.setFetchSize(10000);
-
- insertRecord4Redirect();
- query4Redirect();
- sessionEnableRedirect.close();
+ query();
session.close();
}
- private static void createAndDropContinuousQueries()
- throws StatementExecutionException, IoTDBConnectionException {
- session.executeNonQueryStatement(
- "CREATE CONTINUOUS QUERY cq1 "
- + "BEGIN SELECT max_value(s1) INTO temperature_max FROM root.sg1.* "
- + "GROUP BY time(10s) END");
- session.executeNonQueryStatement(
- "CREATE CONTINUOUS QUERY cq2 "
- + "BEGIN SELECT count(s2) INTO temperature_cnt FROM root.sg1.* "
- + "GROUP BY time(10s), level=1 END");
- session.executeNonQueryStatement(
- "CREATE CONTINUOUS QUERY cq3 "
- + "RESAMPLE EVERY 20s FOR 20s "
- + "BEGIN SELECT avg(s3) INTO temperature_avg FROM root.sg1.* "
- + "GROUP BY time(10s), level=1 END");
- session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq1");
- session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq2");
- session.executeNonQueryStatement("DROP CONTINUOUS QUERY cq3");
- }
-
private static void createTimeseries()
throws IoTDBConnectionException, StatementExecutionException {
@@ -138,229 +71,6 @@ public class SessionExample {
session.createTimeseries(
ROOT_SG1_D1_S1, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY);
}
- if (!session.checkTimeseriesExists(ROOT_SG1_D1_S2)) {
- session.createTimeseries(
- ROOT_SG1_D1_S2, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY);
- }
- if (!session.checkTimeseriesExists(ROOT_SG1_D1_S3)) {
- session.createTimeseries(
- ROOT_SG1_D1_S3, TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY);
- }
-
- // create timeseries with tags and attributes
- if (!session.checkTimeseriesExists(ROOT_SG1_D1_S4)) {
- Map<String, String> tags = new HashMap<>();
- tags.put("tag1", "v1");
- Map<String, String> attributes = new HashMap<>();
- attributes.put("description", "v1");
- session.createTimeseries(
- ROOT_SG1_D1_S4,
- TSDataType.INT64,
- TSEncoding.RLE,
- CompressionType.SNAPPY,
- null,
- tags,
- attributes,
- "temperature");
- }
-
- // create timeseries with SDT property, SDT will take place when flushing
- if (!session.checkTimeseriesExists(ROOT_SG1_D1_S5)) {
- // COMPDEV is required
- // COMPMAXTIME and COMPMINTIME are optional and their unit is ms
- Map<String, String> props = new HashMap<>();
- props.put("LOSS", "sdt");
- props.put("COMPDEV", "0.01");
- props.put("COMPMINTIME", "2");
- props.put("COMPMAXTIME", "10");
- session.createTimeseries(
- ROOT_SG1_D1_S5,
- TSDataType.INT64,
- TSEncoding.RLE,
- CompressionType.SNAPPY,
- props,
- null,
- null,
- null);
- }
- }
-
- private static void createMultiTimeseries()
- throws IoTDBConnectionException, StatementExecutionException {
-
- if (!session.checkTimeseriesExists("root.sg1.d2.s1")
- && !session.checkTimeseriesExists("root.sg1.d2.s2")) {
- List<String> paths = new ArrayList<>();
- paths.add("root.sg1.d2.s1");
- paths.add("root.sg1.d2.s2");
- List<TSDataType> tsDataTypes = new ArrayList<>();
- tsDataTypes.add(TSDataType.INT64);
- tsDataTypes.add(TSDataType.INT64);
- List<TSEncoding> tsEncodings = new ArrayList<>();
- tsEncodings.add(TSEncoding.RLE);
- tsEncodings.add(TSEncoding.RLE);
- List<CompressionType> compressionTypes = new ArrayList<>();
- compressionTypes.add(CompressionType.SNAPPY);
- compressionTypes.add(CompressionType.SNAPPY);
-
- List<Map<String, String>> tagsList = new ArrayList<>();
- Map<String, String> tags = new HashMap<>();
- tags.put("unit", "kg");
- tagsList.add(tags);
- tagsList.add(tags);
-
- List<Map<String, String>> attributesList = new ArrayList<>();
- Map<String, String> attributes = new HashMap<>();
- attributes.put("minValue", "1");
- attributes.put("maxValue", "100");
- attributesList.add(attributes);
- attributesList.add(attributes);
-
- List<String> alias = new ArrayList<>();
- alias.add("weight1");
- alias.add("weight2");
-
- session.createMultiTimeseries(
- paths, tsDataTypes, tsEncodings, compressionTypes, null, tagsList, attributesList, alias);
- }
- }
-
- private static void createTemplate()
- throws IoTDBConnectionException, StatementExecutionException, IOException {
-
- Template template = new Template("template1", false);
- MeasurementNode mNodeS1 =
- new MeasurementNode("s1", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY);
- MeasurementNode mNodeS2 =
- new MeasurementNode("s2", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY);
- MeasurementNode mNodeS3 =
- new MeasurementNode("s3", TSDataType.INT64, TSEncoding.RLE, CompressionType.SNAPPY);
-
- template.addToTemplate(mNodeS1);
- template.addToTemplate(mNodeS2);
- template.addToTemplate(mNodeS3);
-
- session.createSchemaTemplate(template);
- session.setSchemaTemplate("template1", "root.sg1");
- }
-
- private static void insertRecord() throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = ROOT_SG1_D1;
- List<String> measurements = new ArrayList<>();
- List<TSDataType> types = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
-
- for (long time = 0; time < 100; time++) {
- List<Object> values = new ArrayList<>();
- values.add(1L);
- values.add(2L);
- values.add(3L);
- session.insertRecord(deviceId, time, measurements, types, values);
- }
- }
-
- private static void insertRecord4Redirect()
- throws IoTDBConnectionException, StatementExecutionException {
- for (int i = 0; i < 6; i++) {
- for (int j = 0; j < 2; j++) {
- String deviceId = "root.redirect" + i + ".d" + j;
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- List<TSDataType> types = new ArrayList<>();
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
-
- for (long time = 0; time < 5; time++) {
- List<Object> values = new ArrayList<>();
- values.add(1L + time);
- values.add(2L + time);
- values.add(3L + time);
- session.insertRecord(deviceId, time, measurements, types, values);
- }
- }
- }
- }
-
- private static void insertStrRecord()
- throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = ROOT_SG1_D1;
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
-
- for (long time = 0; time < 10; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
- session.insertRecord(deviceId, time, measurements, values);
- }
- }
-
- private static void insertRecordInObject()
- throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = ROOT_SG1_D1;
- List<String> measurements = new ArrayList<>();
- List<TSDataType> types = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
-
- for (long time = 0; time < 100; time++) {
- session.insertRecord(deviceId, time, measurements, types, 1L, 1L, 1L);
- }
- }
-
- private static void insertRecords() throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = ROOT_SG1_D1;
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- List<String> deviceIds = new ArrayList<>();
- List<List<String>> measurementsList = new ArrayList<>();
- List<List<Object>> valuesList = new ArrayList<>();
- List<Long> timestamps = new ArrayList<>();
- List<List<TSDataType>> typesList = new ArrayList<>();
-
- for (long time = 0; time < 500; time++) {
- List<Object> values = new ArrayList<>();
- List<TSDataType> types = new ArrayList<>();
- values.add(1L);
- values.add(2L);
- values.add(3L);
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
- types.add(TSDataType.INT64);
-
- deviceIds.add(deviceId);
- measurementsList.add(measurements);
- valuesList.add(values);
- typesList.add(types);
- timestamps.add(time);
- if (time != 0 && time % 100 == 0) {
- session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
- deviceIds.clear();
- measurementsList.clear();
- valuesList.clear();
- typesList.clear();
- timestamps.clear();
- }
- }
-
- session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
}
/**
@@ -381,125 +91,52 @@ public class SessionExample {
// only measurementId and data type in MeasurementSchema take effects in Tablet
List<MeasurementSchema> schemaList = new ArrayList<>();
schemaList.add(new MeasurementSchema("s1", TSDataType.INT64));
- schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
- schemaList.add(new MeasurementSchema("s3", TSDataType.INT64));
-
- Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100);
+ // schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
+ // schemaList.add(new MeasurementSchema("s3", TSDataType.INT64));
- // Method 1 to add tablet data
- long timestamp = System.currentTimeMillis();
-
- for (long row = 0; row < 100; row++) {
- int rowIndex = tablet.rowSize++;
- tablet.addTimestamp(rowIndex, timestamp);
- for (int s = 0; s < 3; s++) {
- long value = new Random().nextLong();
- tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value);
- }
- if (tablet.rowSize == tablet.getMaxRowNumber()) {
- session.insertTablet(tablet, true);
- tablet.reset();
- }
- timestamp++;
- }
-
- if (tablet.rowSize != 0) {
- session.insertTablet(tablet);
- tablet.reset();
- }
+ Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 10000);
+ int cnt = 1;
// Method 2 to add tablet data
long[] timestamps = tablet.timestamps;
Object[] values = tablet.values;
- for (long time = 0; time < 100; time++) {
+ for (long time = 0; time < 1000000; time++) {
int row = tablet.rowSize++;
timestamps[row] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = i;
- }
+ long[] sensor = (long[]) values[0];
+ sensor[0] = new Random().nextLong();
if (tablet.rowSize == tablet.getMaxRowNumber()) {
session.insertTablet(tablet, true);
tablet.reset();
+ System.out.printf("========= %d =========\n", cnt);
+ cnt++;
}
}
- if (tablet.rowSize != 0) {
- session.insertTablet(tablet);
- tablet.reset();
- }
- }
-
- private static void insertTabletWithNullValues()
- throws IoTDBConnectionException, StatementExecutionException {
- /*
- * A Tablet example:
- * device1
- * time s1, s2, s3
- * 1, null, 1, 1
- * 2, 2, null, 2
- * 3, 3, 3, null
- */
- // The schema of measurements of one device
- // only measurementId and data type in MeasurementSchema take effects in Tablet
- List<MeasurementSchema> schemaList = new ArrayList<>();
- schemaList.add(new MeasurementSchema("s1", TSDataType.INT64));
- schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
- schemaList.add(new MeasurementSchema("s3", TSDataType.INT64));
-
- Tablet tablet = new Tablet(ROOT_SG1_D1, schemaList, 100);
-
- // Method 1 to add tablet data
- tablet.initBitMaps();
-
- long timestamp = System.currentTimeMillis();
- for (long row = 0; row < 100; row++) {
- int rowIndex = tablet.rowSize++;
- tablet.addTimestamp(rowIndex, timestamp);
- for (int s = 0; s < 3; s++) {
- long value = new Random().nextLong();
- // mark null value
- if (row % 3 == s) {
- tablet.bitMaps[s].mark((int) row);
- }
- tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value);
- }
+ for (long time = 5000000; time < 10000000; time++) {
+ int row = tablet.rowSize++;
+ timestamps[row] = time;
+ long[] sensor = (long[]) values[0];
+ sensor[0] = new Random().nextLong();
if (tablet.rowSize == tablet.getMaxRowNumber()) {
session.insertTablet(tablet, true);
tablet.reset();
+ System.out.printf("========= %d =========\n", cnt);
+ cnt++;
}
- timestamp++;
}
- if (tablet.rowSize != 0) {
- session.insertTablet(tablet);
- tablet.reset();
- }
-
- // Method 2 to add tablet data
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
- BitMap[] bitMaps = new BitMap[schemaList.size()];
- for (int s = 0; s < 3; s++) {
- bitMaps[s] = new BitMap(tablet.getMaxRowNumber());
- }
- tablet.bitMaps = bitMaps;
-
- for (long time = 0; time < 100; time++) {
+ for (long time = 1000000; time < 5000000; time++) {
int row = tablet.rowSize++;
timestamps[row] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor = (long[]) values[i];
- // mark null value
- if (row % 3 == i) {
- bitMaps[i].mark(row);
- }
- sensor[row] = i;
- }
+ long[] sensor = (long[]) values[0];
+ sensor[0] = new Random().nextLong();
if (tablet.rowSize == tablet.getMaxRowNumber()) {
session.insertTablet(tablet, true);
tablet.reset();
+ System.out.printf("========= %d =========\n", cnt);
+ cnt++;
}
}
@@ -509,294 +146,15 @@ public class SessionExample {
}
}
- private static void insertTablets() throws IoTDBConnectionException, StatementExecutionException {
- // The schema of measurements of one device
- // only measurementId and data type in MeasurementSchema take effects in Tablet
- List<MeasurementSchema> schemaList = new ArrayList<>();
- schemaList.add(new MeasurementSchema("s1", TSDataType.INT64));
- schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
- schemaList.add(new MeasurementSchema("s3", TSDataType.INT64));
-
- Tablet tablet1 = new Tablet(ROOT_SG1_D1, schemaList, 100);
- Tablet tablet2 = new Tablet("root.sg1.d2", schemaList, 100);
- Tablet tablet3 = new Tablet("root.sg1.d3", schemaList, 100);
-
- Map<String, Tablet> tabletMap = new HashMap<>();
- tabletMap.put(ROOT_SG1_D1, tablet1);
- tabletMap.put("root.sg1.d2", tablet2);
- tabletMap.put("root.sg1.d3", tablet3);
-
- // Method 1 to add tablet data
- long timestamp = System.currentTimeMillis();
- for (long row = 0; row < 100; row++) {
- int row1 = tablet1.rowSize++;
- int row2 = tablet2.rowSize++;
- int row3 = tablet3.rowSize++;
- tablet1.addTimestamp(row1, timestamp);
- tablet2.addTimestamp(row2, timestamp);
- tablet3.addTimestamp(row3, timestamp);
- for (int i = 0; i < 3; i++) {
- long value = new Random().nextLong();
- tablet1.addValue(schemaList.get(i).getMeasurementId(), row1, value);
- tablet2.addValue(schemaList.get(i).getMeasurementId(), row2, value);
- tablet3.addValue(schemaList.get(i).getMeasurementId(), row3, value);
- }
- if (tablet1.rowSize == tablet1.getMaxRowNumber()) {
- session.insertTablets(tabletMap, true);
- tablet1.reset();
- tablet2.reset();
- tablet3.reset();
- }
- timestamp++;
- }
-
- if (tablet1.rowSize != 0) {
- session.insertTablets(tabletMap, true);
- tablet1.reset();
- tablet2.reset();
- tablet3.reset();
- }
-
- // Method 2 to add tablet data
- long[] timestamps1 = tablet1.timestamps;
- Object[] values1 = tablet1.values;
- long[] timestamps2 = tablet2.timestamps;
- Object[] values2 = tablet2.values;
- long[] timestamps3 = tablet3.timestamps;
- Object[] values3 = tablet3.values;
-
- for (long time = 0; time < 100; time++) {
- int row1 = tablet1.rowSize++;
- int row2 = tablet2.rowSize++;
- int row3 = tablet3.rowSize++;
- timestamps1[row1] = time;
- timestamps2[row2] = time;
- timestamps3[row3] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor1 = (long[]) values1[i];
- sensor1[row1] = i;
- long[] sensor2 = (long[]) values2[i];
- sensor2[row2] = i;
- long[] sensor3 = (long[]) values3[i];
- sensor3[row3] = i;
- }
- if (tablet1.rowSize == tablet1.getMaxRowNumber()) {
- session.insertTablets(tabletMap, true);
-
- tablet1.reset();
- tablet2.reset();
- tablet3.reset();
- }
- }
-
- if (tablet1.rowSize != 0) {
- session.insertTablets(tabletMap, true);
- tablet1.reset();
- tablet2.reset();
- tablet3.reset();
- }
- }
-
- private static void selectInto() throws IoTDBConnectionException, StatementExecutionException {
- session.executeNonQueryStatement(
- "select s1, s2, s3 into into_s1, into_s2, into_s3 from root.sg1.d1");
-
- try (SessionDataSet dataSet =
- session.executeQueryStatement("select into_s1, into_s2, into_s3 from root.sg1.d1")) {
- System.out.println(dataSet.getColumnNames());
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- private static void deleteData() throws IoTDBConnectionException, StatementExecutionException {
- String path = ROOT_SG1_D1_S1;
- long deleteTime = 99;
- session.deleteData(path, deleteTime);
- }
-
- private static void deleteTimeseries()
- throws IoTDBConnectionException, StatementExecutionException {
- List<String> paths = new ArrayList<>();
- paths.add(ROOT_SG1_D1_S1);
- paths.add(ROOT_SG1_D1_S2);
- paths.add(ROOT_SG1_D1_S3);
- session.deleteTimeseries(paths);
- }
-
private static void query() throws IoTDBConnectionException, StatementExecutionException {
- try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg1.d1")) {
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- private static void query4Redirect()
- throws IoTDBConnectionException, StatementExecutionException {
- String selectPrefix = "select * from root.redirect";
- for (int i = 0; i < 6; i++) {
- try (SessionDataSet dataSet =
- sessionEnableRedirect.executeQueryStatement(selectPrefix + i + ".d1")) {
-
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- for (int i = 0; i < 6; i++) {
- try (SessionDataSet dataSet =
- sessionEnableRedirect.executeQueryStatement(
- selectPrefix + i + ".d1 where time >= 1 and time < 10")) {
-
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- for (int i = 0; i < 6; i++) {
- try (SessionDataSet dataSet =
- sessionEnableRedirect.executeQueryStatement(
- selectPrefix + i + ".d1 where time >= 1 and time < 10 align by device")) {
-
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- for (int i = 0; i < 6; i++) {
- try (SessionDataSet dataSet =
- sessionEnableRedirect.executeQueryStatement(
- selectPrefix
- + i
- + ".d1 where time >= 1 and time < 10 and root.redirect"
- + i
- + ".d1.s1 > 1")) {
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
- }
-
- private static void queryWithTimeout()
- throws IoTDBConnectionException, StatementExecutionException {
- try (SessionDataSet dataSet =
- session.executeQueryStatement("select * from root.sg1.d1", 2000)) {
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- private static void rawDataQuery() throws IoTDBConnectionException, StatementExecutionException {
- List<String> paths = new ArrayList<>();
- paths.add(ROOT_SG1_D1_S1);
- paths.add(ROOT_SG1_D1_S2);
- paths.add(ROOT_SG1_D1_S3);
- long startTime = 10L;
- long endTime = 200L;
-
- try (SessionDataSet dataSet = session.executeRawDataQuery(paths, startTime, endTime)) {
-
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024);
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
- }
- }
- }
-
- private static void lastDataQuery() throws IoTDBConnectionException, StatementExecutionException {
- List<String> paths = new ArrayList<>();
- paths.add(ROOT_SG1_D1_S1);
- paths.add(ROOT_SG1_D1_S2);
- paths.add(ROOT_SG1_D1_S3);
- try (SessionDataSet sessionDataSet = session.executeLastDataQuery(paths, 3)) {
- System.out.println(sessionDataSet.getColumnNames());
- sessionDataSet.setFetchSize(1024);
- while (sessionDataSet.hasNext()) {
- System.out.println(sessionDataSet.next());
- }
- }
- }
-
- private static void queryByIterator()
- throws IoTDBConnectionException, StatementExecutionException {
- try (SessionDataSet dataSet = session.executeQueryStatement("select * from root.sg1.d1")) {
-
- DataIterator iterator = dataSet.iterator();
- System.out.println(dataSet.getColumnNames());
- dataSet.setFetchSize(1024); // default is 10000
- while (iterator.next()) {
- StringBuilder builder = new StringBuilder();
- // get time
- builder.append(iterator.getLong(1)).append(",");
- // get second column
- if (!iterator.isNull(2)) {
- builder.append(iterator.getLong(2)).append(",");
- } else {
- builder.append("null").append(",");
- }
-
- // get third column
- if (!iterator.isNull(ROOT_SG1_D1_S2)) {
- builder.append(iterator.getLong(ROOT_SG1_D1_S2)).append(",");
- } else {
- builder.append("null").append(",");
- }
-
- // get forth column
- if (!iterator.isNull(4)) {
- builder.append(iterator.getLong(4)).append(",");
- } else {
- builder.append("null").append(",");
- }
-
- // get fifth column
- if (!iterator.isNull(ROOT_SG1_D1_S4)) {
- builder.append(iterator.getObject(ROOT_SG1_D1_S4));
- } else {
- builder.append("null");
- }
-
- System.out.println(builder);
- }
- }
- }
-
- private static void nonQuery() throws IoTDBConnectionException, StatementExecutionException {
- session.executeNonQueryStatement("insert into root.sg1.d1(timestamp,s1) values(200, 1)");
- }
-
- private static void setTimeout() throws StatementExecutionException {
- Session tempSession = new Session(LOCAL_HOST, 6667, "root", "root", 10000, 20000);
- tempSession.setQueryTimeout(60000);
- }
-
- private static void createClusterSession() throws IoTDBConnectionException {
- ArrayList<String> nodeList = new ArrayList<>();
- nodeList.add("127.0.0.1:6669");
- nodeList.add("127.0.0.1:6667");
- nodeList.add("127.0.0.1:6668");
- Session clusterSession = new Session(nodeList, "root", "root");
- clusterSession.open();
- clusterSession.close();
+ long startTime = System.nanoTime();
+ for (int i = 0; i < 100; i++) {
+ // session.executeQueryStatement("select count(s1) from root.sg1.d1");
+ session.executeQueryStatement(
+ "select count(s1) from root.sg1.d1 group by ([5, 4523623), 511ms)");
+ }
+ // 旧 Time: 125.357620 ms Time: 124.753079 ms
+ // 新 Time: 167.694330 ms
+ System.out.printf("Time: %f ms", (System.nanoTime() - startTime) / 1e8);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 4d9c447db8..cef455546e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -368,13 +368,13 @@ public class IoTDBConfig {
private int avgSeriesPointNumberThreshold = 100000;
/** Enable inner space compaction for sequence files */
- private boolean enableSeqSpaceCompaction = true;
+ private boolean enableSeqSpaceCompaction = false;
/** Enable inner space compaction for unsequence files */
- private boolean enableUnseqSpaceCompaction = true;
+ private boolean enableUnseqSpaceCompaction = false;
/** Compact the unsequence files into the overlapped sequence files */
- private boolean enableCrossSpaceCompaction = true;
+ private boolean enableCrossSpaceCompaction = false;
/**
* The strategy of inner space compaction task. There are just one inner space compaction strategy