You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2020/06/29 03:40:43 UTC
[incubator-iotdb] branch master updated: [IOTDB-736] Query
performance tracing (#1399)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a019a90 [IOTDB-736] Query performance tracing (#1399)
a019a90 is described below
commit a019a90a33e46b1d6aaf2f1eb45df8c52458e6f6
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Mon Jun 29 11:40:34 2020 +0800
[IOTDB-736] Query performance tracing (#1399)
* Add tracing on/off statement
---
.../resources/conf/iotdb-engine.properties | 3 +-
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 10 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 57 ++++++---
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 2 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 22 +++-
.../engine/storagegroup/StorageGroupProcessor.java | 2 +-
.../main/java/org/apache/iotdb/db/qp/Planner.java | 1 +
.../apache/iotdb/db/qp/constant/SQLConstant.java | 3 +
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 7 ++
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../iotdb/db/qp/logical/sys/TracingOperator.java | 36 ++++++
.../iotdb/db/qp/physical/sys/TracingPlan.java | 44 +++++++
.../iotdb/db/qp/strategy/LogicalGenerator.java | 15 +++
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 5 +
.../db/query/control/QueryResourceManager.java | 78 ++++++++++--
.../iotdb/db/query/control/TracingManager.java | 138 +++++++++++++++++++++
.../db/query/dataset/AlignByDeviceDataSet.java | 10 ++
.../iotdb/db/query/executor/QueryRouter.java | 24 ++--
.../db/query/executor/RawDataQueryExecutor.java | 2 +-
.../iotdb/db/query/reader/series/SeriesReader.java | 56 ++++++---
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 67 ++++++++--
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 1 -
.../iotdb/db/integration/IoTDBTracingTest.java | 67 ++++++++++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +
25 files changed, 588 insertions(+), 72 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 8f3d59a..de9f17b 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -370,7 +370,8 @@ enable_performance_stat=false
performance_stat_display_interval=60000
# The memory used for performance_stat in kb.
performance_stat_memory_in_kb=20
-
+# Is performance tracing enable
+enable_performance_tracing=false
####################
diff --git a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4 b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
index f78e854..2c39fc5 100644
--- a/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
+++ b/server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
@@ -74,6 +74,8 @@ statement
| SHOW CHILD PATHS prefixPath? #showChildPaths
| SHOW DEVICES prefixPath? #showDevices
| SHOW MERGE #showMergeStatus
+ | TRACING ON #tracingOn
+ | TRACING OFF #tracingOff
| COUNT TIMESERIES prefixPath? (GROUP BY LEVEL OPERATOR_EQ INT)? #countTimeseries
| COUNT NODES prefixPath LEVEL OPERATOR_EQ INT #countNodes
| LOAD CONFIGURATION (MINUS GLOBAL)? #loadConfigurationStatement
@@ -671,10 +673,18 @@ USING
: U S I N G
;
+TRACING
+ : T R A C I N G
+ ;
+
ON
: O N
;
+OFF
+ : O F F
+ ;
+
DROP
: D R O P
;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 5c2e085..6d3eafc 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -21,6 +21,10 @@ package org.apache.iotdb.db.conf;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_ROOT;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+import java.io.File;
+import java.time.ZoneId;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.merge.selector.MergeFileStrategy;
import org.apache.iotdb.db.exception.LoadConfigurationException;
@@ -33,11 +37,6 @@ import org.apache.iotdb.tsfile.fileSystem.FSType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.time.ZoneId;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
public class IoTDBConfig {
/* Names of Watermark methods */
@@ -56,7 +55,7 @@ public class IoTDBConfig {
// for path like: root.sg1.d1."1.2.3" or root.sg1.d1.'1.2.3', only occurs in the end of the path and only occurs once
private static final String NODE_WITH_QUOTATION_MARK_MATCHER =
- "[" + PATH_SEPARATOR + "][\"|\']" + ID_MATCHER +"(" + NODE_MATCHER+ ")*[\"|\']";
+ "[" + PATH_SEPARATOR + "][\"|\']" + ID_MATCHER + "(" + NODE_MATCHER + ")*[\"|\']";
public static final Pattern PATH_PATTERN = Pattern
.compile(PATH_ROOT + "(" + NODE_MATCHER + ")+(" + NODE_WITH_QUOTATION_MARK_MATCHER + ")?");
@@ -173,22 +172,29 @@ public class IoTDBConfig {
/**
* System directory, including version file for each storage group and metadata
*/
- private String systemDir = "data" + File.separator + "system";
+ private String systemDir = baseDir + File.separator + IoTDBConstant.SYSTEM_FOLDER_NAME;
/**
* Schema directory, including storage set of values.
*/
- private String schemaDir = "data" + File.separator + "system" + File.separator + "schema";
+ private String schemaDir = baseDir + File.separator + IoTDBConstant.SYSTEM_FOLDER_NAME
+ + File.separator + IoTDBConstant.SCHEMA_FOLDER_NAME;
/**
* Sync directory, including the lock file, uuid file, device owner map
*/
- private String syncDir = "data" + File.separator + "system" + File.separator + "sync";
+ private String syncDir = baseDir + File.separator + IoTDBConstant.SYSTEM_FOLDER_NAME
+ + File.separator + IoTDBConstant.SYNC_FOLDER_NAME;
+
+ /**
+ * Performance tracing directory, stores performance tracing files
+ */
+ private String tracingDir = baseDir + File.separator + IoTDBConstant.TRACING_FOLDER_NAME;
/**
* Query directory, stores temporary files of query
*/
- private String queryDir = "data" + File.separator + "query";
+ private String queryDir = baseDir + File.separator + IoTDBConstant.QUERY_FOLDER_NAME;
/**
* Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"};
@@ -203,7 +209,7 @@ public class IoTDBConfig {
/**
* Wal directory.
*/
- private String walFolder = "data" + File.separator + "wal";
+ private String walFolder = baseDir + File.separator + "wal";
/**
* Maximum MemTable number in MemTable pool.
@@ -332,6 +338,11 @@ public class IoTDBConfig {
private boolean enablePerformanceStat = false;
/**
+ * Is performance tracing enable.
+ */
+ private boolean enablePerformanceTracing = false;
+
+ /**
* The display of stat performance interval in ms.
*/
private long performanceStatDisplayInterval = 60000;
@@ -386,7 +397,8 @@ public class IoTDBConfig {
private TSDataType floatingStringInferType = TSDataType.FLOAT;
/**
- * register time series as which type when receiving the Literal NaN. Values can be DOUBLE, FLOAT or TEXT
+ * register time series as which type when receiving the Literal NaN. Values can be DOUBLE, FLOAT
+ * or TEXT
*/
private TSDataType nanStringInferType = TSDataType.DOUBLE;
@@ -557,8 +569,8 @@ public class IoTDBConfig {
/**
* default TTL for storage groups that are not set TTL by statements, in ms
- * Notice: if this property is changed, previous created storage group which are not set TTL will
- * also be affected.
+ * Notice: if this property is changed, previous created storage group which are not set TTL will also be
+ * affected.
*/
private long defaultTTL = Long.MAX_VALUE;
@@ -684,6 +696,7 @@ public class IoTDBConfig {
schemaDir = addHomeDir(schemaDir);
syncDir = addHomeDir(syncDir);
walFolder = addHomeDir(walFolder);
+ tracingDir = addHomeDir(tracingDir);
if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) {
String hdfsDir = getHdfsDir();
@@ -856,6 +869,14 @@ public class IoTDBConfig {
this.syncDir = syncDir;
}
+ public String getTracingDir() {
+ return tracingDir;
+ }
+
+ void setTracingDir(String tracingDir) {
+ this.tracingDir = tracingDir;
+ }
+
public String getQueryDir() {
return queryDir;
}
@@ -1136,6 +1157,14 @@ public class IoTDBConfig {
this.enablePerformanceStat = enablePerformanceStat;
}
+ public boolean isEnablePerformanceTracing() {
+ return enablePerformanceTracing;
+ }
+
+ public void setEnablePerformanceTracing(boolean enablePerformanceTracing) {
+ this.enablePerformanceTracing = enablePerformanceTracing;
+ }
+
public long getPerformanceStatDisplayInterval() {
return performanceStatDisplayInterval;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 5a7f5d9..8bc924a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -97,6 +97,8 @@ public class IoTDBConstant {
public static final String SCHEMA_FOLDER_NAME = "schema";
public static final String SYNC_FOLDER_NAME = "sync";
public static final String QUERY_FOLDER_NAME = "query";
+ public static final String TRACING_FOLDER_NAME = "tracing";
+ public static final String TRACING_LOG = "tracing.txt";
// mqtt
public static final String ENABLE_MQTT = "enable_mqtt_service";
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 289e38f..8b21986 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -202,6 +202,9 @@ public class IoTDBDescriptor {
conf.setSyncDir(
FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SYNC_FOLDER_NAME);
+ conf.setTracingDir(FilePathUtils
+ .regularizePath(conf.getBaseDir() + IoTDBConstant.TRACING_FOLDER_NAME));
+
conf.setQueryDir(
FilePathUtils.regularizePath(conf.getBaseDir()) + IoTDBConstant.QUERY_FOLDER_NAME);
@@ -325,6 +328,10 @@ public class IoTDBDescriptor {
.parseBoolean(properties.getProperty("enable_performance_stat",
Boolean.toString(conf.isEnablePerformanceStat())).trim()));
+ conf.setEnablePerformanceTracing(Boolean
+ .parseBoolean(properties.getProperty("enable_performance_tracing",
+ Boolean.toString(conf.isEnablePerformanceTracing())).trim()));
+
conf.setPerformanceStatDisplayInterval(Long
.parseLong(properties.getProperty("performance_stat_display_interval",
Long.toString(conf.getPerformanceStatDisplayInterval())).trim()));
@@ -498,12 +505,15 @@ public class IoTDBDescriptor {
conf.setAutoCreateSchemaEnabled(
Boolean.parseBoolean(properties.getProperty("enable_auto_create_schema",
Boolean.toString(conf.isAutoCreateSchemaEnabled()).trim())));
- conf.setBooleanStringInferType(TSDataType.valueOf(properties.getProperty("boolean_string_infer_type",
- conf.getBooleanStringInferType().toString())));
- conf.setIntegerStringInferType(TSDataType.valueOf(properties.getProperty("integer_string_infer_type",
- conf.getIntegerStringInferType().toString())));
- conf.setFloatingStringInferType(TSDataType.valueOf(properties.getProperty("floating_string_infer_type",
- conf.getFloatingStringInferType().toString())));
+ conf.setBooleanStringInferType(
+ TSDataType.valueOf(properties.getProperty("boolean_string_infer_type",
+ conf.getBooleanStringInferType().toString())));
+ conf.setIntegerStringInferType(
+ TSDataType.valueOf(properties.getProperty("integer_string_infer_type",
+ conf.getIntegerStringInferType().toString())));
+ conf.setFloatingStringInferType(
+ TSDataType.valueOf(properties.getProperty("floating_string_infer_type",
+ conf.getFloatingStringInferType().toString())));
conf.setNanStringInferType(TSDataType.valueOf(properties.getProperty("nan_string_infer_type",
conf.getNanStringInferType().toString())));
conf.setDefaultStorageGroupLevel(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 009acac..d53b7e8 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -74,8 +74,8 @@ import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.OutOfTTLException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.metadata.mnode.MNode;
+import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index 1a39a74..1601102 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -97,6 +97,7 @@ public class Planner {
case MOVE_FILE:
case FLUSH:
case MERGE:
+ case TRACING:
case CLEAR_CACHE:
case NULL:
case SHOW_MERGE_STATUS:
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
index 33d0dbe..3ae7b21 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
@@ -153,6 +153,7 @@ public class SQLConstant {
public static final int TOK_SHOW_MERGE_STATUS = 87;
public static final int TOK_CREATE_SCHEMA_SNAPSHOT = 88;
+ public static final int TOK_TRACING = 91;
public static final Map<Integer, String> tokenSymbol = new HashMap<>();
public static final Map<Integer, String> tokenNames = new HashMap<>();
@@ -223,6 +224,8 @@ public class SQLConstant {
tokenNames.put(TOK_MOVE_FILE, "TOK_MOVE_FILE");
tokenNames.put(TOK_SHOW_MERGE_STATUS, "TOK_SHOW_MERGE_STATUS");
+
+ tokenNames.put(TOK_TRACING, "TOK_TRACING");
}
static {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index d2dfb28..adb0b04 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -192,6 +192,9 @@ public class PlanExecutor implements IPlanExecutor {
case FULL_MERGE:
operateMerge((MergePlan) plan);
return true;
+ case TRACING:
+ operateTracing((TracingPlan) plan);
+ return true;
case CLEAR_CACHE:
operateClearCache((ClearCachePlan) plan);
return true;
@@ -223,6 +226,10 @@ public class PlanExecutor implements IPlanExecutor {
mManager.createMTreeSnapshot();
}
+ private void operateTracing(TracingPlan plan) {
+ IoTDBDescriptor.getInstance().getConfig().setEnablePerformanceTracing(plan.isTracingOn());
+ }
+
private void operateFlush(FlushPlan plan) throws StorageGroupNotSetException {
if (plan.getPaths() == null) {
StorageEngine.getInstance().syncCloseAllProcessor();
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
index 704288e..42f5345 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
@@ -76,6 +76,6 @@ public abstract class Operator {
GRANT_WATERMARK_EMBEDDING, REVOKE_WATERMARK_EMBEDDING,
TTL, DELETE_STORAGE_GROUP, LOAD_CONFIGURATION, SHOW, LOAD_FILES, REMOVE_FILE, MOVE_FILE, LAST, GROUP_BY_FILL,
ALTER_TIMESERIES, FLUSH, MERGE, FULL_MERGE, CLEAR_CACHE,
- SHOW_MERGE_STATUS, CREATE_SCHEMA_SNAPSHOT
+ SHOW_MERGE_STATUS, CREATE_SCHEMA_SNAPSHOT, TRACING
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java
new file mode 100644
index 0000000..b4ee85d
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.qp.logical.sys;
+
+import org.apache.iotdb.db.qp.logical.RootOperator;
+
+public class TracingOperator extends RootOperator {
+
+ private boolean isTracingon;
+
+ public TracingOperator(int tokenIntType, boolean isTracingon) {
+ super(tokenIntType);
+ this.isTracingon = isTracingon;
+ operatorType = OperatorType.TRACING;
+ }
+
+ public boolean isTracingon() {
+ return isTracingon;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java
new file mode 100644
index 0000000..26ed373
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.qp.physical.sys;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class TracingPlan extends PhysicalPlan {
+
+ private boolean isTracingOn;
+
+ public TracingPlan(boolean isTracingOn) {
+ super(false, OperatorType.TRACING);
+ this.isTracingOn = isTracingOn;
+ }
+
+ @Override
+ public List<Path> getPaths() {
+ return new ArrayList<>();
+ }
+
+ public boolean isTracingOn() {
+ return isTracingOn;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index 2c37e4d..edc2b9c 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -69,6 +69,7 @@ import org.apache.iotdb.db.qp.logical.sys.ShowMergeStatusOperator;
import org.apache.iotdb.db.qp.logical.sys.ShowOperator;
import org.apache.iotdb.db.qp.logical.sys.ShowTTLOperator;
import org.apache.iotdb.db.qp.logical.sys.ShowTimeSeriesOperator;
+import org.apache.iotdb.db.qp.logical.sys.TracingOperator;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.AliasContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.AlignByDeviceClauseContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.AlterUserContext;
@@ -154,6 +155,8 @@ import org.apache.iotdb.db.qp.strategy.SqlBaseParser.SoffsetClauseContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.SuffixPathContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.TagClauseContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.TimeIntervalContext;
+import org.apache.iotdb.db.qp.strategy.SqlBaseParser.TracingOffContext;
+import org.apache.iotdb.db.qp.strategy.SqlBaseParser.TracingOnContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.TypeClauseContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.UnsetTTLStatementContext;
import org.apache.iotdb.db.qp.strategy.SqlBaseParser.UpdateStatementContext;
@@ -226,6 +229,18 @@ public class LogicalGenerator extends SqlBaseBaseListener {
}
@Override
+ public void enterTracingOn(TracingOnContext ctx) {
+ super.enterTracingOn(ctx);
+ initializedOperator = new TracingOperator(SQLConstant.TOK_TRACING, true);
+ }
+
+ @Override
+ public void enterTracingOff(TracingOffContext ctx) {
+ super.enterTracingOff(ctx);
+ initializedOperator = new TracingOperator(SQLConstant.TOK_TRACING, false);
+ }
+
+ @Override
public void enterMerge(MergeContext ctx) {
super.enterMerge(ctx);
initializedOperator = new MergeOperator(SQLConstant.TOK_MERGE);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index 76edb63..a72fcb0 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -60,6 +60,7 @@ import org.apache.iotdb.db.qp.logical.sys.ShowChildPathsOperator;
import org.apache.iotdb.db.qp.logical.sys.ShowDevicesOperator;
import org.apache.iotdb.db.qp.logical.sys.ShowTTLOperator;
import org.apache.iotdb.db.qp.logical.sys.ShowTimeSeriesOperator;
+import org.apache.iotdb.db.qp.logical.sys.TracingOperator;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.AggregationPlan;
import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan;
@@ -96,6 +97,7 @@ import org.apache.iotdb.db.qp.physical.sys.ShowPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowPlan.ShowContentType;
import org.apache.iotdb.db.qp.physical.sys.ShowTTLPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.TracingPlan;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -182,6 +184,9 @@ public class PhysicalGenerator {
case FLUSH:
FlushOperator flushOperator = (FlushOperator) operator;
return new FlushPlan(flushOperator.isSeq(), flushOperator.getStorageGroupList());
+ case TRACING:
+ TracingOperator tracingOperator = (TracingOperator) operator;
+ return new TracingPlan(tracingOperator.isTracingon());
case QUERY:
QueryOperator query = (QueryOperator) operator;
return transformQuery(query);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index 1fb4976..2f65c75 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -18,8 +18,23 @@
*/
package org.apache.iotdb.db.query.control;
+import java.io.File;
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -27,13 +42,8 @@ import org.apache.iotdb.db.query.externalsort.serialize.IExternalSortFileDeseria
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <p>
@@ -47,6 +57,16 @@ public class QueryResourceManager {
private AtomicLong queryIdAtom = new AtomicLong();
private QueryFileManager filePathsManager;
+ private static final Logger logger = LoggerFactory.getLogger(QueryResourceManager.class);
+ // record the total number and size of chunks for each query id
+ private Map<Long, Long> chunkNumMap = new ConcurrentHashMap<>();
+ // chunk size represents the number of time-value points in the chunk
+ private Map<Long, Long> chunkSizeMap = new ConcurrentHashMap<>();
+ // record the distinct tsfiles for each query id
+ // Just store weak references here in case GC failed for those objects
+ private Map<Long, Set<WeakReference<TsFileResource>>> seqFileNumMap = new ConcurrentHashMap<>();
+ private Map<Long, Set<WeakReference<TsFileResource>>> unseqFileNumMap = new ConcurrentHashMap<>();
+ private IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
/**
* Record temporary files used for external sorting.
* <p>
@@ -74,6 +94,14 @@ public class QueryResourceManager {
return queryId;
}
+ public Map<Long, Long> getChunkNumMap() {
+ return chunkNumMap;
+ }
+
+ public Map<Long, Long> getChunkSizeMap() {
+ return chunkSizeMap;
+ }
+
/**
* register temporary file generated by external sort for resource release.
*
@@ -85,13 +113,23 @@ public class QueryResourceManager {
externalSortFileMap.computeIfAbsent(queryId, x -> new ArrayList<>()).add(deserializer);
}
-
public QueryDataSource getQueryDataSource(Path selectedPath,
QueryContext context, Filter filter) throws StorageEngineException, QueryProcessException {
SingleSeriesExpression singleSeriesExpression = new SingleSeriesExpression(selectedPath,
filter);
- return StorageEngine.getInstance().query(singleSeriesExpression, context, filePathsManager);
+ QueryDataSource queryDataSource = StorageEngine.getInstance()
+ .query(singleSeriesExpression, context, filePathsManager);
+ // calculate the distinct number of seq and unseq tsfiles
+ if (config.isEnablePerformanceTracing()) {
+ seqFileNumMap.computeIfAbsent(context.getQueryId(), k -> new HashSet<>())
+ .addAll((queryDataSource.getSeqResources().stream().map(r -> new WeakReference<>(r))
+ .collect(Collectors.toSet())));
+ unseqFileNumMap.computeIfAbsent(context.getQueryId(), k -> new HashSet<>())
+ .addAll((queryDataSource.getUnseqResources().stream().map(r -> new WeakReference<>(r))
+ .collect(Collectors.toSet())));
+ }
+ return queryDataSource;
}
/**
@@ -99,6 +137,28 @@ public class QueryResourceManager {
* query tokens created by this jdbc request must be cleared.
*/
public void endQuery(long queryId) throws StorageEngineException {
+ try {
+ if (config.isEnablePerformanceTracing()) {
+ boolean isprinted = false;
+ if (seqFileNumMap.get(queryId) != null && unseqFileNumMap.get(queryId) != null) {
+ TracingManager.getInstance().writeTsFileInfo(queryId, seqFileNumMap.remove(queryId).size(),
+ unseqFileNumMap.remove(queryId).size());
+ isprinted = true;
+ }
+ if (chunkNumMap.get(queryId) != null && chunkSizeMap.get(queryId) != null) {
+ TracingManager.getInstance()
+ .writeChunksInfo(queryId, chunkNumMap.remove(queryId), chunkSizeMap.remove(queryId));
+ }
+ if (isprinted) {
+ TracingManager.getInstance().writeEndTime(queryId);
+ }
+ }
+ } catch (IOException e) {
+ logger.error(
+ "Error while writing performance info to {}, {}",
+ config.getTracingDir() + File.separator + IoTDBConstant.TRACING_LOG, e.getMessage());
+ }
+
// close file stream of external sort files, and delete
if (externalSortFileMap.get(queryId) != null) {
for (IExternalSortFileDeserializer deserializer : externalSortFileMap.get(queryId)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/TracingManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/TracingManager.java
new file mode 100644
index 0000000..7c5aed3
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/TracingManager.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.query.control;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TracingManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(TracingManager.class);
+ private BufferedWriter writer;
+
+ public TracingManager(String dirName, String logFileName) {
+ File tracingDir = SystemFileFactory.INSTANCE.getFile(dirName);
+ if (!tracingDir.exists()) {
+ if (tracingDir.mkdirs()) {
+ logger.info("create performance folder {}.", tracingDir);
+ } else {
+ logger.info("create performance folder {} failed.", tracingDir);
+ }
+ }
+ File logFile = SystemFileFactory.INSTANCE.getFile(dirName + File.separator + logFileName);
+
+ FileWriter fileWriter = null;
+ try {
+ fileWriter = new FileWriter(logFile, true);
+ } catch (IOException e) {
+ logger.error("Meeting error while creating TracingManager: {}", e);
+ }
+ writer = new BufferedWriter(fileWriter);
+ }
+
+ public static TracingManager getInstance() {
+ return TracingManagerHelper.INSTANCE;
+ }
+
+ public void writeQueryInfo(long queryId, String statement, int pathsNum) throws IOException {
+ StringBuilder builder = new StringBuilder("-----------------------------\n");
+ builder.append("Query Id: ").append(queryId)
+ .append(" - Query Statement: ").append(statement)
+ .append("\nQuery Id: ").append(queryId)
+ .append(" - Start time: ")
+ .append(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(System.currentTimeMillis()))
+ .append("\nQuery Id: ").append(queryId)
+ .append(" - Number of series paths: ").append(pathsNum)
+ .append("\n");
+ writer.write(builder.toString());
+ }
+
+ // for align by device query
+ public void writeQueryInfo(long queryId, String statement) throws IOException {
+ StringBuilder builder = new StringBuilder("-----------------------------\n");
+ builder.append("Query Id: ").append(queryId)
+ .append(" - Query Statement: ").append(statement)
+ .append("\nQuery Id: ").append(queryId)
+ .append(" - Start time: ")
+ .append(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(System.currentTimeMillis()))
+ .append("\n");
+ writer.write(builder.toString());
+ }
+
+ public void writePathsNum(long queryId, int pathsNum) throws IOException {
+ StringBuilder builder = new StringBuilder("Query Id: ").append(queryId)
+ .append(" - Number of series paths: ").append(pathsNum)
+ .append("\n");
+ writer.write(builder.toString());
+ }
+
+ public void writeTsFileInfo(long queryId, int seqFileNum, int unseqFileNum) throws IOException {
+ // to avoid the disorder info of multi query
+ // add query id as prefix of each info
+ StringBuilder builder = new StringBuilder("Query Id: ").append(queryId)
+ .append(" - Number of tsfiles: ").append(seqFileNum + unseqFileNum)
+ .append("\nQuery Id: ").append(queryId)
+ .append(" - Number of sequence files: ").append(seqFileNum)
+ .append("\nQuery Id: ").append(queryId)
+ .append(" - Number of unsequence files: ").append(unseqFileNum)
+ .append("\n");
+ writer.write(builder.toString());
+ }
+
+ public void writeChunksInfo(long queryId, long totalChunkNum, long totalChunkSize)
+ throws IOException {
+ StringBuilder builder = new StringBuilder("Query Id: ").append(queryId)
+ .append(" - Number of chunks: ").append(totalChunkNum)
+ .append("\nQuery Id: ").append(queryId)
+ .append(" - Average size of chunks: ").append(totalChunkSize / totalChunkNum)
+ .append("\n");
+ writer.write(builder.toString());
+ }
+
+ public void writeEndTime(long queryId) throws IOException {
+ StringBuilder builder = new StringBuilder("Query Id: ").append(queryId)
+ .append(" - End time: ")
+ .append(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(System.currentTimeMillis()))
+ .append("\n");
+ writer.write(builder.toString());
+ writer.flush();
+ }
+
+ public void close() throws IOException {
+ writer.close();
+ }
+
+ private static class TracingManagerHelper {
+
+ private static final TracingManager INSTANCE = new TracingManager(
+ IoTDBDescriptor.getInstance().getConfig().getTracingDir(),
+ IoTDBConstant.TRACING_LOG);
+
+ private TracingManagerHelper() {
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java
index 2988800..e351b91 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/AlignByDeviceDataSet.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
@@ -75,6 +76,7 @@ public class AlignByDeviceDataSet extends QueryDataSet {
private QueryDataSet currentDataSet;
private Iterator<String> deviceIterator;
private List<String> executeColumns;
+ private int pathsNum = 0;
public AlignByDeviceDataSet(AlignByDevicePlan alignByDevicePlan, QueryContext context,
IQueryRouter queryRouter) throws MetadataException {
@@ -110,6 +112,10 @@ public class AlignByDeviceDataSet extends QueryDataSet {
this.deviceIterator = devices.iterator();
}
+ public int getPathsNum() {
+ return pathsNum;
+ }
+
protected boolean hasNextWithoutConstraint() throws IOException {
if (curDataSetInitialized && currentDataSet.hasNext()) {
return true;
@@ -148,6 +154,10 @@ public class AlignByDeviceDataSet extends QueryDataSet {
this.expression = deviceToFilterMap.get(currentDevice);
}
+ if (IoTDBDescriptor.getInstance().getConfig().isEnablePerformanceTracing()) {
+ pathsNum += executeColumns.size();
+ }
+
try {
switch (dataSetType) {
case GROUPBYTIME:
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java b/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java
index 4d2070f..08add9f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/QueryRouter.java
@@ -91,9 +91,9 @@ public class QueryRouter implements IQueryRouter {
if (logger.isDebugEnabled()) {
logger.debug("paths:" + aggregationPlan.getPaths()
- + " level:" + aggregationPlan.getLevel()
- + " duplicatePaths:" + aggregationPlan.getDeduplicatedPaths()
- + " deduplicatePaths:" + aggregationPlan.getDeduplicatedAggregations());
+ + " level:" + aggregationPlan.getLevel()
+ + " duplicatePaths:" + aggregationPlan.getDeduplicatedPaths()
+ + " deduplicatePaths:" + aggregationPlan.getDeduplicatedAggregations());
}
IExpression expression = aggregationPlan.getExpression();
@@ -126,7 +126,7 @@ public class QueryRouter implements IQueryRouter {
@Override
public QueryDataSet groupBy(GroupByTimePlan groupByTimePlan, QueryContext context)
- throws QueryFilterOptimizationException, StorageEngineException, QueryProcessException, IOException {
+ throws QueryFilterOptimizationException, StorageEngineException, QueryProcessException, IOException {
if (logger.isDebugEnabled()) {
logger.debug("paths:" + groupByTimePlan.getPaths() + " level:" + groupByTimePlan.getLevel());
@@ -142,7 +142,7 @@ public class QueryRouter implements IQueryRouter {
List<Path> selectedSeries = groupByTimePlan.getDeduplicatedPaths();
GlobalTimeExpression timeExpression = new GlobalTimeExpression(
- new GroupByFilter(unit, slidingStep, startTime, endTime));
+ new GroupByFilter(unit, slidingStep, startTime, endTime));
if (expression == null) {
expression = timeExpression;
@@ -152,7 +152,7 @@ public class QueryRouter implements IQueryRouter {
// optimize expression to an executable one
IExpression optimizedExpression = ExpressionOptimizer.getInstance()
- .optimize(expression, selectedSeries);
+ .optimize(expression, selectedSeries);
groupByTimePlan.setExpression(optimizedExpression);
if (optimizedExpression.getType() == ExpressionType.GLOBAL_TIME) {
@@ -181,9 +181,9 @@ public class QueryRouter implements IQueryRouter {
}
protected GroupByTimeDataSet groupByLevelWithoutTimeIntervalDataSet(QueryContext context, GroupByTimePlan plan,
- GroupByEngineDataSet dataSet)
- throws StorageEngineException, QueryProcessException, IOException {
- return new GroupByTimeDataSet(context, plan, dataSet);
+ GroupByEngineDataSet dataSet)
+ throws StorageEngineException, QueryProcessException, IOException {
+ return new GroupByTimeDataSet(context, plan, dataSet);
}
@Override
@@ -208,15 +208,15 @@ public class QueryRouter implements IQueryRouter {
@Override
public QueryDataSet groupByFill(GroupByTimeFillPlan groupByFillPlan, QueryContext context)
- throws QueryFilterOptimizationException, StorageEngineException, QueryProcessException, IOException {
+ throws QueryFilterOptimizationException, StorageEngineException, QueryProcessException, IOException {
GroupByEngineDataSet groupByEngineDataSet = (GroupByEngineDataSet) groupBy(groupByFillPlan, context);
return new GroupByFillDataSet(groupByFillPlan.getDeduplicatedPaths(), groupByFillPlan.getDeduplicatedDataTypes(),
- groupByEngineDataSet, groupByFillPlan.getFillType(), context, groupByFillPlan);
+ groupByEngineDataSet, groupByFillPlan.getFillType(), context, groupByFillPlan);
}
@Override
public QueryDataSet lastQuery(LastQueryPlan lastQueryPlan, QueryContext context)
- throws StorageEngineException, QueryProcessException, IOException {
+ throws StorageEngineException, QueryProcessException, IOException {
LastQueryExecutor lastQueryExecutor = new LastQueryExecutor(lastQueryPlan);
return lastQueryExecutor.execute(context, lastQueryPlan);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java
index 2a4d63f..f541666 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/RawDataQueryExecutor.java
@@ -138,7 +138,7 @@ public class RawDataQueryExecutor {
}
protected IReaderByTimestamp getReaderByTimestamp(Path path, Set<String> allSensors, TSDataType dataType,
- QueryContext context) throws StorageEngineException, QueryProcessException {
+ QueryContext context) throws StorageEngineException, QueryProcessException {
return new SeriesReaderByTimestamp(path, allSensors,
dataType, context,
QueryResourceManager.getInstance().getQueryDataSource(path, context, null), null);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index bab80df..2fa3717 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -18,9 +18,18 @@
*/
package org.apache.iotdb.db.query.reader.series;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader;
import org.apache.iotdb.db.utils.FileLoaderUtils;
@@ -37,10 +46,6 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.basic.UnaryFilter;
import org.apache.iotdb.tsfile.read.reader.IPageReader;
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
public class SeriesReader {
private final Path seriesPath;
@@ -73,7 +78,7 @@ public class SeriesReader {
private final List<TimeseriesMetadata> seqTimeSeriesMetadata = new LinkedList<>();
private final PriorityQueue<TimeseriesMetadata> unSeqTimeSeriesMetadata =
new PriorityQueue<>(Comparator.comparingLong(
- timeSeriesMetadata -> timeSeriesMetadata.getStatistics().getStartTime()));
+ timeSeriesMetadata -> timeSeriesMetadata.getStatistics().getStartTime()));
/*
* chunk cache
@@ -100,7 +105,8 @@ public class SeriesReader {
private boolean hasCachedNextOverlappedPage;
private BatchData cachedBatchData;
- public SeriesReader(Path seriesPath, Set<String> allSensors, TSDataType dataType, QueryContext context,
+ public SeriesReader(Path seriesPath, Set<String> allSensors, TSDataType dataType,
+ QueryContext context,
QueryDataSource dataSource, Filter timeFilter, Filter valueFilter, TsFileFilter fileFilter) {
this.seriesPath = seriesPath;
this.allSensors = allSensors;
@@ -166,11 +172,11 @@ public class SeriesReader {
Statistics fileStatistics = firstTimeSeriesMetadata.getStatistics();
return !seqTimeSeriesMetadata.isEmpty()
- && fileStatistics.getEndTime()
- >= seqTimeSeriesMetadata.get(0).getStatistics().getStartTime()
+ && fileStatistics.getEndTime()
+ >= seqTimeSeriesMetadata.get(0).getStatistics().getStartTime()
|| !unSeqTimeSeriesMetadata.isEmpty()
- && fileStatistics.getEndTime()
- >= unSeqTimeSeriesMetadata.peek().getStatistics().getStartTime();
+ && fileStatistics.getEndTime()
+ >= unSeqTimeSeriesMetadata.peek().getStatistics().getStartTime();
}
Statistics currentFileStatistics() {
@@ -251,6 +257,7 @@ public class SeriesReader {
unpackOneTimeSeriesMetadata(firstTimeSeriesMetadata);
firstTimeSeriesMetadata = null;
}
+
if (init && firstChunkMetadata == null && !cachedChunkMetadata.isEmpty()) {
firstChunkMetadata = cachedChunkMetadata.poll();
}
@@ -258,7 +265,22 @@ public class SeriesReader {
private void unpackOneTimeSeriesMetadata(TimeseriesMetadata timeSeriesMetadata)
throws IOException {
- cachedChunkMetadata.addAll(FileLoaderUtils.loadChunkMetadataList(timeSeriesMetadata));
+ List<ChunkMetadata> chunkMetadataList = FileLoaderUtils
+ .loadChunkMetadataList(timeSeriesMetadata);
+ // try to calculate the total number of chunk and time-value points in chunk
+ if (IoTDBDescriptor.getInstance().getConfig().isEnablePerformanceTracing()) {
+ QueryResourceManager queryResourceManager = QueryResourceManager.getInstance();
+ queryResourceManager.getChunkNumMap()
+ .compute(context.getQueryId(),
+ (k, v) -> v == null ? chunkMetadataList.size() : v + chunkMetadataList.size());
+
+ long totalChunkSize = chunkMetadataList.stream()
+ .mapToLong(chunkMetadata -> chunkMetadata.getStatistics().getCount()).sum();
+ queryResourceManager.getChunkSizeMap()
+ .compute(context.getQueryId(), (k, v) -> v == null ? totalChunkSize : v + totalChunkSize);
+ }
+
+ cachedChunkMetadata.addAll(chunkMetadataList);
}
boolean isChunkOverlapped() throws IOException {
@@ -431,7 +453,9 @@ public class SeriesReader {
firstPageReader = null;
}
- /** This method should only be used when the method isPageOverlapped() return true. */
+ /**
+ * This method should only be used when the method isPageOverlapped() return true.
+ */
BatchData nextPage() throws IOException {
if (!hasNextPage()) {
@@ -499,7 +523,7 @@ public class SeriesReader {
if (valueFilter == null
|| valueFilter.satisfy(
- timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
+ timeValuePair.getTimestamp(), timeValuePair.getValue().getValue())) {
cachedBatchData.putAnObject(
timeValuePair.getTimestamp(), timeValuePair.getValue().getValue());
}
@@ -584,9 +608,9 @@ public class SeriesReader {
/**
* unpack all overlapped seq/unseq files and find the first TimeSeriesMetadata
*
- * <p>Because there may be too many files in the scenario used by the user, we cannot open all the
- * chunks at once, which may cause OOM, so we can only unpack one file at a time when needed. This
- * approach is likely to be ubiquitous, but it keeps the system running smoothly
+ * <p>Because there may be too many files in the scenario used by the user, we cannot open all
+ * the chunks at once, which may cause OOM, so we can only unpack one file at a time when needed.
+ * This approach is likely to be ubiquitous, but it keeps the system running smoothly
*/
private void tryToUnpackAllOverlappedFilesToTimeSeriesMetadata() throws IOException {
/*
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index dc3ffa6..9b717ef 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.StatMonitor;
+import org.apache.iotdb.db.query.control.TracingManager;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
@@ -152,6 +153,7 @@ public class IoTDB implements IoTDBMBean {
public void shutdown() throws Exception {
logger.info("Deactivating IoTDB...");
MManager.getInstance().clear();
+ TracingManager.getInstance().close();
registerManager.shutdownAll();
JMXService.deregisterMBean(mbeanName);
logger.info("IoTDB is deactivated.");
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 8f62241..7aec45c 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -18,6 +18,24 @@
*/
package org.apache.iotdb.db.service;
+import static org.apache.iotdb.db.conf.IoTDBConfig.PATH_PATTERN;
+import static org.apache.iotdb.db.qp.physical.sys.ShowPlan.ShowContentType.TIMESERIES;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.sql.SQLException;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.AuthorityChecker;
@@ -43,11 +61,24 @@ import org.apache.iotdb.db.qp.executor.IPlanExecutor;
import org.apache.iotdb.db.qp.executor.PlanExecutor;
import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
-import org.apache.iotdb.db.qp.physical.crud.*;
+import org.apache.iotdb.db.qp.physical.crud.AggregationPlan;
+import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan;
import org.apache.iotdb.db.qp.physical.crud.AlignByDevicePlan.MeasurementType;
-import org.apache.iotdb.db.qp.physical.sys.*;
+import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
+import org.apache.iotdb.db.qp.physical.crud.LastQueryPlan;
+import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
+import org.apache.iotdb.db.qp.physical.sys.AuthorPlan;
+import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan;
+import org.apache.iotdb.db.qp.physical.sys.DeleteTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan;
+import org.apache.iotdb.db.qp.physical.sys.ShowPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.control.TracingManager;
+import org.apache.iotdb.db.query.dataset.AlignByDeviceDataSet;
import org.apache.iotdb.db.query.dataset.NonAlignEngineDataSet;
import org.apache.iotdb.db.query.dataset.RawQueryDataSetWithoutValueFilter;
import org.apache.iotdb.db.tools.watermark.GroupedLSBWatermarkEncoder;
@@ -88,7 +119,8 @@ import static org.apache.iotdb.db.qp.physical.sys.ShowPlan.ShowContentType.TIMES
*/
public class TSServiceImpl implements TSIService.Iface, ServerContext {
- private static final Logger auditLogger = LoggerFactory.getLogger(IoTDBConstant.AUDIT_LOGGER_NAME);
+ private static final Logger auditLogger = LoggerFactory
+ .getLogger(IoTDBConstant.AUDIT_LOGGER_NAME);
private static final Logger logger = LoggerFactory.getLogger(TSServiceImpl.class);
private static final String INFO_NOT_LOGIN = "{}: Not login.";
private static final int MAX_SIZE =
@@ -237,9 +269,9 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
@Override
public TSStatus closeOperation(TSCloseOperationReq req) {
if (auditLogger.isDebugEnabled()) {
- auditLogger.debug("{}: receive close operation from Session {}", IoTDBConstant.GLOBAL_DB_NAME,
- currSessionId.get());
- }
+ auditLogger.debug("{}: receive close operation from Session {}", IoTDBConstant.GLOBAL_DB_NAME,
+ currSessionId.get());
+ }
if (!checkLogin(req.getSessionId())) {
auditLogger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME);
return RpcUtils.getStatus(TSStatusCode.NOT_LOGIN_ERROR);
@@ -389,7 +421,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
} catch (SQLParserException e) {
logger.error("Error occurred when executing {}, check metadata error: ", statement, e);
result.add(RpcUtils.getStatus(
- TSStatusCode.SQL_PARSE_ERROR, ERROR_PARSING_SQL + " " + statement + " " + e.getMessage()));
+ TSStatusCode.SQL_PARSE_ERROR,
+ ERROR_PARSING_SQL + " " + statement + " " + e.getMessage()));
return false;
} catch (QueryProcessException e) {
logger.info(
@@ -494,7 +527,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
* AuthorPlan
*/
private TSExecuteStatementResp internalExecuteQueryStatement(String statement,
- long statementId, PhysicalPlan plan, int fetchSize, String username) {
+ long statementId, PhysicalPlan plan, int fetchSize, String username) throws IOException {
auditLogger.info("Session {} execute Query: {}", currSessionId.get(), statement);
long startTime = System.currentTimeMillis();
long queryId = -1;
@@ -518,6 +551,13 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
resp.setOperationType(plan.getOperatorType().toString());
// generate the queryId for the operation
queryId = generateQueryId(true);
+ if (plan instanceof QueryPlan && config.isEnablePerformanceTracing()) {
+ if (!(plan instanceof AlignByDevicePlan)) {
+ TracingManager.getInstance().writeQueryInfo(queryId, statement, plan.getPaths().size());
+ } else {
+ TracingManager.getInstance().writeQueryInfo(queryId, statement);
+ }
+ }
// put it into the corresponding Set
statementId2QueryId.computeIfAbsent(statementId, k -> new HashSet<>()).add(queryId);
@@ -540,6 +580,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
resp.setQueryId(queryId);
+ if (plan instanceof AlignByDevicePlan && config.isEnablePerformanceTracing()) {
+ TracingManager.getInstance()
+ .writePathsNum(queryId, ((AlignByDeviceDataSet) newDataSet).getPathsNum());
+ }
+
if (enableMetric) {
long endTime = System.currentTimeMillis();
SqlArgument sqlArgument = new SqlArgument(resp, plan, statement, startTime, endTime);
@@ -659,8 +704,10 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
// Last Query should return different respond instead of the static one
// because the query dataset and query id is different although the header of last query is same.
return StaticResps.LAST_RESP.deepCopy();
- } else if (plan instanceof AggregationPlan && ((AggregationPlan)plan).getLevel() >= 0) {
- Map<String, Long> finalPaths = FilePathUtils.getPathByLevel(((AggregationPlan)plan).getDeduplicatedPaths(), ((AggregationPlan)plan).getLevel(), null);
+ } else if (plan instanceof AggregationPlan && ((AggregationPlan) plan).getLevel() >= 0) {
+ Map<String, Long> finalPaths = FilePathUtils
+ .getPathByLevel(((AggregationPlan) plan).getDeduplicatedPaths(),
+ ((AggregationPlan) plan).getLevel(), null);
for (Map.Entry<String, Long> entry : finalPaths.entrySet()) {
respColumns.add("count(" + entry.getKey() + ")");
columnsTypes.add(TSDataType.INT64.toString());
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 245c4f4..85b0a82 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -129,7 +129,6 @@ public class FileLoaderUtils {
return timeSeriesMetadata.loadChunkMetadataList();
}
-
/**
* load all page readers in one chunk that satisfying the timeFilter
*
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTracingTest.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTracingTest.java
new file mode 100644
index 0000000..4d9ecf3
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTracingTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class IoTDBTracingTest {
+
+ @BeforeClass
+ public static void setUp(){
+ EnvironmentUtils.closeStatMonitor();
+ EnvironmentUtils.envSetUp();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void tracingTest() throws ClassNotFoundException {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ Assert.assertEquals(config.isEnablePerformanceTracing(), false);
+
+ statement.execute("tracing on");
+ Assert.assertEquals(config.isEnablePerformanceTracing(), true);
+
+ statement.execute("tracing off");
+ Assert.assertEquals(config.isEnablePerformanceTracing(), false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 9ae753a..ee7f045 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -42,6 +42,7 @@ import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.control.TracingManager;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
@@ -137,6 +138,9 @@ public class EnvironmentUtils {
// close metadata
MManager.getInstance().clear();
+ // close tracing
+ TracingManager.getInstance().close();
+
// delete all directory
cleanAllDir();
@@ -161,6 +165,8 @@ public class EnvironmentUtils {
cleanDir(config.getWalFolder());
// delete query
cleanDir(config.getQueryDir());
+ // delete tracing
+ cleanDir(config.getTracingDir());
cleanDir(config.getBaseDir());
// delete data files
for (String dataDir : config.getDataDirs()) {