You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/01/12 08:21:30 UTC
[iotdb] branch lmh/addMetricDoc created (now 88f17c6a3b)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 88f17c6a3b update Metric-Tool.md (en)
This branch includes the following new commits:
new 8319a0b5b5 remove SubRawQuery
new 76d51bf29c remove useless config
new 0588ed06a4 update metric name (operation -> statement_execution)
new 6f578dd78b update metric name (operation -> statement_execution)(en)
new 3b7e26c8e8 Merge remote-tracking branch 'origin/master' into lmh/addMetricDoc
new 74908eccce add more metrics
new dd57589f5d delete metric dispatcher:SCHEDULE
new e92addd65c fix data_exchange_count tag name
new 398bd8280b update Metric-Tool.md (zh)
new 88f17c6a3b update Metric-Tool.md (en)
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 06/10: add more metrics
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 74908eccce3d893ee15075b4cc623361847c5c71
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Jan 10 16:40:53 2023 +0800
add more metrics
---
.../db/mpp/execution/exchange/LocalSinkHandle.java | 2 +-
.../mpp/execution/exchange/LocalSourceHandle.java | 4 +-
.../execution/exchange/MPPDataExchangeManager.java | 14 ++-
.../db/mpp/execution/exchange/SinkHandle.java | 6 +-
.../db/mpp/execution/exchange/SourceHandle.java | 14 ++-
...tricSet.java => DataExchangeCostMetricSet.java} | 10 +-
.../db/mpp/metric/DataExchangeCountMetricSet.java | 119 +++++++++++++++++++++
.../iotdb/db/mpp/metric/QueryMetricsManager.java | 13 +--
.../db/service/metrics/DataNodeMetricsHelper.java | 6 +-
9 files changed, 155 insertions(+), 33 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
index 0a1d666409..8afb07fe6e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
@@ -33,7 +33,7 @@ import java.util.List;
import java.util.Optional;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SINK_HANDLE_SEND_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SINK_HANDLE_SEND_TSBLOCK_LOCAL;
public class LocalSinkHandle implements ISinkHandle {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java
index 6487a20399..f47dea04d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java
@@ -37,8 +37,8 @@ import java.nio.ByteBuffer;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeManager.createFullIdFrom;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_LOCAL;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_GET_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_GET_TSBLOCK_LOCAL;
public class LocalSourceHandle implements ISourceHandle {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java
index 885447d0c0..b3d89c0340 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java
@@ -54,9 +54,12 @@ import java.util.concurrent.ExecutorService;
import java.util.function.Supplier;
import static org.apache.iotdb.db.mpp.common.FragmentInstanceId.createFullId;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.GET_DATA_BLOCK_TASK_SERVER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_SERVER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.GET_DATA_BLOCK_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.GET_DATA_BLOCK_NUM_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.SEND_NEW_DATA_BLOCK_NUM_SERVER;
public class MPPDataExchangeManager implements IMPPDataExchangeManager {
@@ -118,6 +121,8 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager {
} finally {
QUERY_METRICS.recordDataExchangeCost(
GET_DATA_BLOCK_TASK_SERVER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(
+ GET_DATA_BLOCK_NUM_SERVER, req.getEndSequenceId() - req.getStartSequenceId());
}
}
@@ -150,6 +155,8 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager {
} finally {
QUERY_METRICS.recordDataExchangeCost(
ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_SERVER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER, e.getEndSequenceId() - e.getStartSequenceId());
}
}
@@ -192,6 +199,7 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager {
} finally {
QUERY_METRICS.recordDataExchangeCost(
SEND_NEW_DATA_BLOCK_EVENT_TASK_SERVER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(SEND_NEW_DATA_BLOCK_NUM_SERVER, e.getBlockSizes().size());
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java
index e2494af0fb..139387d510 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java
@@ -51,8 +51,9 @@ import java.util.concurrent.ExecutorService;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.common.FragmentInstanceId.createFullId;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SINK_HANDLE_SEND_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SINK_HANDLE_SEND_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.SEND_NEW_DATA_BLOCK_NUM_CALLER;
import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
public class SinkHandle implements ISinkHandle {
@@ -428,6 +429,7 @@ public class SinkHandle implements ISinkHandle {
} finally {
QUERY_METRICS.recordDataExchangeCost(
SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(SEND_NEW_DATA_BLOCK_NUM_CALLER, blockSizes.size());
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java
index 2e1a4b8a6d..d164e9d1de 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java
@@ -51,10 +51,12 @@ import java.util.concurrent.ExecutorService;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeManager.createFullIdFrom;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.GET_DATA_BLOCK_TASK_CALLER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_CALLER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_REMOTE;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_GET_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.GET_DATA_BLOCK_TASK_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_GET_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.GET_DATA_BLOCK_NUM_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER;
public class SourceHandle implements ISourceHandle {
@@ -467,7 +469,7 @@ public class SourceHandle implements ISourceHandle {
tsBlocks.addAll(resp.getTsBlocks());
logger.debug("[EndPullTsBlocksFromRemote] Count:{}", tsBlockNum);
- QUERY_METRICS.recordDataBlockNum(tsBlockNum);
+ QUERY_METRICS.recordDataBlockNum(GET_DATA_BLOCK_NUM_CALLER, tsBlockNum);
executorService.submit(
new SendAcknowledgeDataBlockEventTask(startSequenceId, endSequenceId));
synchronized (SourceHandle.this) {
@@ -581,6 +583,8 @@ public class SourceHandle implements ISourceHandle {
} finally {
QUERY_METRICS.recordDataExchangeCost(
ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER, endSequenceId - startSequenceId);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCostMetricSet.java
similarity index 94%
rename from server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeMetricSet.java
rename to server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCostMetricSet.java
index 004b640d71..aab7d1ba5d 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeMetricSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCostMetricSet.java
@@ -30,7 +30,7 @@ import org.apache.iotdb.metrics.utils.MetricType;
import java.util.HashMap;
import java.util.Map;
-public class DataExchangeMetricSet implements IMetricSet {
+public class DataExchangeCostMetricSet implements IMetricSet {
private static final String metric = Metric.DATA_EXCHANGE_COST.toString();
@@ -170,19 +170,12 @@ public class DataExchangeMetricSet implements IMetricSet {
"server"));
}
- public static final String GET_DATA_BLOCK_NUM = "get_data_block_num";
-
@Override
public void bindTo(AbstractMetricService metricService) {
for (MetricInfo metricInfo : metricInfoMap.values()) {
metricService.getOrCreateTimer(
metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
}
- metricService.getOrCreateHistogram(
- Metric.DATA_EXCHANGE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- GET_DATA_BLOCK_NUM);
}
@Override
@@ -190,6 +183,5 @@ public class DataExchangeMetricSet implements IMetricSet {
for (MetricInfo metricInfo : metricInfoMap.values()) {
metricService.remove(MetricType.TIMER, metric, metricInfo.getTagsInArray());
}
- metricService.remove(MetricType.HISTOGRAM, metric, Tag.NAME.toString(), GET_DATA_BLOCK_NUM);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
new file mode 100644
index 0000000000..5d8ef839f4
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.metric;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricInfo;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataExchangeCountMetricSet implements IMetricSet {
+
+ private static final String metric = Metric.DATA_EXCHANGE_COUNT.toString();
+
+ public static final Map<String, MetricInfo> metricInfoMap = new HashMap<>();
+
+ public static final String SEND_NEW_DATA_BLOCK_NUM_CALLER = "send_new_data_block_num_caller";
+ public static final String SEND_NEW_DATA_BLOCK_NUM_SERVER = "send_new_data_block_num_server";
+ public static final String ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER =
+ "on_acknowledge_data_block_num_caller";
+ public static final String ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER =
+ "on_acknowledge_data_block_num_server";
+ public static final String GET_DATA_BLOCK_NUM_CALLER = "get_data_block_num_caller";
+ public static final String GET_DATA_BLOCK_NUM_SERVER = "get_data_block_num_server";
+
+ static {
+ metricInfoMap.put(
+ SEND_NEW_DATA_BLOCK_NUM_CALLER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.OPERATION.toString(),
+ "send_new_data_block_num",
+ Tag.TYPE.toString(),
+ "caller"));
+ metricInfoMap.put(
+ SEND_NEW_DATA_BLOCK_NUM_SERVER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.OPERATION.toString(),
+ "send_new_data_block_num",
+ Tag.TYPE.toString(),
+ "server"));
+ metricInfoMap.put(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.OPERATION.toString(),
+ "on_acknowledge_data_block_num",
+ Tag.TYPE.toString(),
+ "caller"));
+ metricInfoMap.put(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.OPERATION.toString(),
+ "on_acknowledge_data_block_num",
+ Tag.TYPE.toString(),
+ "server"));
+ metricInfoMap.put(
+ GET_DATA_BLOCK_NUM_CALLER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.OPERATION.toString(),
+ "get_data_block_num",
+ Tag.TYPE.toString(),
+ "caller"));
+ metricInfoMap.put(
+ GET_DATA_BLOCK_NUM_SERVER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.OPERATION.toString(),
+ "get_data_block_num",
+ Tag.TYPE.toString(),
+ "server"));
+ }
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ for (MetricInfo metricInfo : metricInfoMap.values()) {
+ metricService.getOrCreateHistogram(
+ metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
+ }
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ for (MetricInfo metricInfo : metricInfoMap.values()) {
+ metricService.remove(MetricType.HISTOGRAM, metric, metricInfo.getTagsInArray());
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
index 206125bb2e..b5bcc5be3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
@@ -27,8 +27,6 @@ import org.apache.iotdb.metrics.utils.MetricLevel;
import java.util.concurrent.TimeUnit;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.GET_DATA_BLOCK_NUM;
-
public class QueryMetricsManager {
private final MetricService metricService = MetricService.getInstance();
@@ -88,7 +86,7 @@ public class QueryMetricsManager {
}
public void recordDataExchangeCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = DataExchangeMetricSet.metricInfoMap.get(stage);
+ MetricInfo metricInfo = DataExchangeCostMetricSet.metricInfoMap.get(stage);
metricService.timer(
costTimeInNanos,
TimeUnit.NANOSECONDS,
@@ -97,13 +95,10 @@ public class QueryMetricsManager {
metricInfo.getTagsInArray());
}
- public void recordDataBlockNum(int num) {
+ public void recordDataBlockNum(String type, int num) {
+ MetricInfo metricInfo = DataExchangeCountMetricSet.metricInfoMap.get(type);
metricService.histogram(
- num,
- Metric.DATA_EXCHANGE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- GET_DATA_BLOCK_NUM);
+ num, metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
}
public void recordTaskQueueTime(String name, long queueTimeInNanos) {
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index 34ded3bb24..074b7345a6 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -20,7 +20,8 @@
package org.apache.iotdb.db.service.metrics;
import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet;
+import org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet;
+import org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet;
import org.apache.iotdb.db.mpp.metric.DriverSchedulerMetricSet;
import org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet;
import org.apache.iotdb.db.mpp.metric.QueryPlanCostMetricSet;
@@ -43,7 +44,8 @@ public class DataNodeMetricsHelper {
MetricService.getInstance().addMetricSet(new SeriesScanCostMetricSet());
MetricService.getInstance().addMetricSet(new QueryExecutionMetricSet());
MetricService.getInstance().addMetricSet(new QueryResourceMetricSet());
- MetricService.getInstance().addMetricSet(new DataExchangeMetricSet());
+ MetricService.getInstance().addMetricSet(new DataExchangeCostMetricSet());
+ MetricService.getInstance().addMetricSet(new DataExchangeCountMetricSet());
MetricService.getInstance().addMetricSet(new DriverSchedulerMetricSet());
}
}
[iotdb] 07/10: delete metric dispatcher:SCHEDULE
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dd57589f5d1db8d1766744cd2b740468b025a26f
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Jan 12 14:48:22 2023 +0800
delete metric dispatcher:SCHEDULE
---
.../java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java | 5 -----
.../java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java | 5 -----
2 files changed, 10 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
index adaf851310..2c3f5d7bc9 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
@@ -34,15 +34,10 @@ public class QueryExecutionMetricSet implements IMetricSet {
public static final Map<String, MetricInfo> metricInfoMap = new HashMap<>();
- public static final String SCHEDULE = "schedule";
public static final String WAIT_FOR_DISPATCH = "wait_for_dispatch";
public static final String DISPATCH_READ = "dispatch_read";
static {
- metricInfoMap.put(
- SCHEDULE,
- new MetricInfo(
- MetricType.TIMER, Metric.DISPATCHER.toString(), Tag.STAGE.toString(), SCHEDULE));
metricInfoMap.put(
WAIT_FOR_DISPATCH,
new MetricInfo(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
index 9cf25c433c..b0326403b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
@@ -81,7 +81,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static org.apache.iotdb.db.mpp.common.DataNodeEndPoints.isSameNode;
-import static org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet.SCHEDULE;
import static org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet.WAIT_FOR_RESULT;
import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetricSet.DISTRIBUTION_PLANNER;
@@ -274,7 +273,6 @@ public class QueryExecution implements IQueryExecution {
}
// TODO: (xingtanzjr) initialize the query scheduler according to configuration
- long startTime = System.nanoTime();
this.scheduler =
new ClusterScheduler(
context,
@@ -286,9 +284,6 @@ public class QueryExecution implements IQueryExecution {
scheduledExecutor,
internalServiceClientManager);
this.scheduler.start();
- if (rawStatement.isQuery()) {
- QUERY_METRICS.recordExecutionCost(SCHEDULE, System.nanoTime() - startTime);
- }
}
// Use LogicalPlanner to do the logical query plan and logical optimization
[iotdb] 03/10: update metric name (operation -> statement_execution)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0588ed06a4bc74eacd1614e7ec262f34184d026c
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Jan 10 10:39:23 2023 +0800
update metric name (operation -> statement_execution)
---
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 75 +++++++++++++++-------
.../commons/service/metric/enums/Operation.java | 40 ------------
2 files changed, 53 insertions(+), 62 deletions(-)
diff --git a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
index b32ade409f..2af7c8756e 100644
--- a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -56,18 +56,19 @@
- `Metric Name`：指标类型名称，比如`logback_events`表示日志事件。
- `Tags`：指标分类，形式为Key-Value对，每个指标下面可以有0到多个分类，常见的Key-Value对：
- - `name = xxx`：被监控对象的名称，是对**业务逻辑**的说明。比如对于`Metric Name = entry_seconds_count`类型的监控项，name的含义是指被监控的业务接口。
- - `type = xxx`：监控指标类型细分，是对**监控指标**本身的说明。比如对于`Metric Name = point`类型的监控项，type的含义是指监控具体是什么类型的点数。
- - `status = xxx`：被监控对象的状态，是对**业务逻辑**的说明。比如对于`Metric Name = Task`类型的监控项可以通过该参数，从而区分被监控对象的状态。
- - `user = xxx`：被监控对象的相关用户，是对**业务逻辑**的说明。比如统计`root`用户的写入总点数。
- - 根据具体情况自定义：比如logback_events_total下有一个level的分类，用来表示特定级别下的日志数量。
+ - `name = xxx`：被监控对象的名称，是对**业务逻辑**的说明。比如对于`Metric Name = entry_seconds_count`类型的监控项，name的含义是指被监控的业务接口。
+ - `type = xxx`：监控指标类型细分，是对**监控指标**本身的说明。比如对于`Metric Name = point`类型的监控项，type的含义是指监控具体是什么类型的点数。
+ - `status = xxx`：被监控对象的状态，是对**业务逻辑**的说明。比如对于`Metric Name = Task`类型的监控项可以通过该参数，从而区分被监控对象的状态。
+ - `user = xxx`：被监控对象的相关用户，是对**业务逻辑**的说明。比如统计`root`用户的写入总点数。
+ - 根据具体情况自定义：比如logback_events_total下有一个level的分类，用来表示特定级别下的日志数量。
- `Metric Level`：**指标管理级别**，默认启动级别为`Core`级别，建议启动级别为`Important级别`，审核严格程度`Core > Important > Normal > All`
- `Core`：系统的核心指标，供**系统内核和运维人员**使用，关乎系统的**性能、稳定性、安全性**，比如实例的状况，系统的负载等。
- `Important`：模块的重要指标，供**运维和测试人员**使用，直接关乎**每个模块的运行状态**，比如合并文件个数、执行情况等。
- `Normal`：模块的一般指标，供**开发人员**使用，方便在出现问题时**定位模块**，比如合并中的特定关键操作情况。
- `All`：模块的全部指标，供**模块开发人员**使用，往往在复现问题的时候使用，从而快速解决问题。
-
+
### 3.2. 监控指标对外获取数据格式
+
- IoTDB 对外提供 JMX、 Prometheus 和 IoTDB 格式的监控指标：
- 对于 JMX ，可以通过```org.apache.iotdb.metrics```获取系统监控指标指标。
- 对于 Prometheus ，可以通过对外暴露的端口获取监控指标的值
@@ -91,6 +92,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| points | database="{{database}}", type="flush" | Gauge | 最新一个刷盘的memtale的点数 |
#### 4.1.2. IoTDB 进程运行状态
+
| Metric | Tags | Type | Description |
| ----------------- | -------------- | --------- | ----------------------------------- |
| process_cpu_load | name="process" | AutoGauge | IoTDB 进程的 CPU 占用率，单位为% |
@@ -100,6 +102,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| process_free_mem | name="memory" | AutoGauge | IoTDB 进程当前剩余可用内存 |
#### 4.1.3. 系统运行状态
+
| Metric | Tags | Type | Description |
| ------------------------------ | ------------- | --------- | ---------------------------------------- |
| sys_cpu_load | name="system" | AutoGauge | 系统的 CPU 占用率，单位为% |
@@ -117,12 +120,14 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
目前 Important 级别的监控指标如下所述：
#### 4.2.1. 集群运行状态
+
| Metric | Tags | Type | Description |
| ------------------------- | ------------------------------------------------- | ----- | ------------------------------ |
| cluster_node_leader_count | name="{{ip}}:{{port}}" | Gauge | 节点上共识组Leader的数量 |
| cluster_node_status | name="{{ip}}:{{port}}",type="ConfigNode/DataNode" | Gauge | 节点的状态，0=Unkonwn 1=online |
#### 4.2.2. 节点统计
+
| Metric | Tags | Type | Description |
| -------- | ------------------------------------------ | --------- | ------------------------------------ |
| quantity | name="database" | AutoGauge | 系统数据库数量 |
@@ -134,6 +139,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| region | name="{{ip}}:{{port}}",type="DataRegion" | Gauge | 分区表中对应节点上 DataRegion 总数量 |
#### 4.2.3. IoT共识协议统计
+
| Metric | Tags | Type | Description |
| ------------- | -------------------------------------------------------------------------------------------- | --------- | -------------------------------- |
| iot_consensus | name="logDispatcher-{{IP}}:{{Port}}", region="{{region}}", type="currentSyncIndex" | AutoGauge | 副本组同步线程的当前同步进度 |
@@ -149,6 +155,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| stage | name="iot_consensus", region="{{region}}", type="syncLogTimePerRequest" | Histogram | 异步回调流程同步日志耗时 |
#### 4.2.4. 缓存统计
+
| Metric | Tags | Type | Description |
| --------- | ---------------------------------- | --------- | ------------------------------------------------------- |
| cache_hit | name="chunk" | AutoGauge | ChunkCache的命中率，单位为% |
@@ -163,20 +170,22 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| cache | name="DataPartition", type="all" | Counter | DataPartition Cache 的访问次数 |
#### 4.2.5. 接口层统计
-| Metric | Tags | Type | Description |
-| --------------------- | ---------------------------------- | --------- | ----------------------------------- |
-| operation | name = "{{name}}" | Histogram | 客户端执行的操作的耗时情况 |
-| entry | name="{{interface}}" | Timer | Client 建立的 Thrift 的耗时情况 |
-| thrift_connections | name="ConfigNodeRPC" | AutoGauge | ConfigNode 的内部 Thrift 连接数 |
-| thrift_connections | name="Internal" | AutoGauge | DataNode 的内部 Thrift 连接数 |
-| thrift_connections | name="MPPDataExchange" | AutoGauge | MPP 框架的内部 Thrift 连接数 |
-| thrift_connections | name="RPC" | AutoGauge | Client 建立的 Thrift 连接数 |
-| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | ConfigNode 的内部活跃 Thrift 连接数 |
-| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | DataNode 的内部活跃 Thrift 连接数 |
-| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | MPP 框架的内部活跃 Thrift 连接数 |
-| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | Client 建立的活跃 Thrift 连接数 |
+
+| Metric | Tags | Type | Description |
+| --------------------- | ---------------------------------------------------- | --------- | ----------------------------------- |
+| statement_execution | interface="{{interface}}", type="{{statement_type}}" | Timer | 客户端执行的操作的耗时情况 |
+| entry | name="{{interface}}" | Timer | Client 建立的 Thrift 的耗时情况 |
+| thrift_connections | name="ConfigNodeRPC" | AutoGauge | ConfigNode 的内部 Thrift 连接数 |
+| thrift_connections | name="Internal" | AutoGauge | DataNode 的内部 Thrift 连接数 |
+| thrift_connections | name="MPPDataExchange" | AutoGauge | MPP 框架的内部 Thrift 连接数 |
+| thrift_connections | name="RPC" | AutoGauge | Client 建立的 Thrift 连接数 |
+| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | ConfigNode 的内部活跃 Thrift 连接数 |
+| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | DataNode 的内部活跃 Thrift 连接数 |
+| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | MPP 框架的内部活跃 Thrift 连接数 |
+| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | Client 建立的活跃 Thrift 连接数 |
#### 4.2.6. 内存统计
+
| Metric | Tags | Type | Description |
| ------ | ------------------------------------ | --------- | ------------------------------------------------- |
| mem | name="database_{{name}}" | AutoGauge | DataNode内对应DataRegion的内存占用，单位为byte |
@@ -186,6 +195,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| mem | name="schema_region_total_remaining" | AutoGauge | 所有SchemaRegion的总内存剩余，单位为byte |
#### 4.2.7. 任务统计
+
| Metric | Tags | Type | Description |
| --------- | ------------------------------------------------- | --------- | ------------------ |
| queue | name="compaction_inner", status="running/waiting" | Gauge | 空间内合并任务数 |
@@ -194,6 +204,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| queue | name="flush",status="running/waiting" | AutoGauge | 刷盘任务数 |
#### 4.2.8. 合并统计
+
| Metric | Tags | Type | Description |
| --------------------- | --------------------------------------------------- | ------- | ------------------ |
| data_written | name="compaction", type="aligned/not-aligned/total" | Counter | 合并时写入量 |
@@ -221,6 +232,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| file_count | name="open_file_handlers" | AutoGauge | IoTDB 进程打开文件数，仅支持Linux和MacOS |
#### 4.2.10. IoTDB 进程统计
+
| Metric | Tags | Type | Description |
| --------------------- | -------------- | --------- | ------------------------------------ |
| process_used_mem | name="memory" | AutoGauge | IoTDB 进程当前使用内存 |
@@ -229,6 +241,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| process_status | name="process" | AutoGauge | IoTDB 进程存活状态，1为存活，0为终止 |
#### 4.2.11. IoTDB 日志统计
+
| Metric | Tags | Type | Description |
| -------------- | ----------------------------------- | ------- | ------------------ |
| logback_events | level="trace/debug/info/warn/error" | Counter | 不同类型的日志个数 |
@@ -243,6 +256,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| jvm_threads_states_threads | state="runnable/blocked/waiting/timed-waiting/new/terminated" | AutoGauge | 当前处于各种状态的线程数 |
#### 4.2.13. JVM GC 统计
+
| Metric | Tags | Type | Description |
| ----------------------------- | ----------------------------------------------------- | --------- | -------------------------------------- |
| jvm_gc_pause | action="end of major GC/end of minor GC",cause="xxxx" | Timer | 不同原因的Young GC/Full GC的次数与耗时 |
@@ -255,6 +269,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| jvm_gc_memory_allocated_bytes | | Counter | GC分配内存正向增长累计值 |
#### 4.2.14. JVM 内存统计
+
| Metric | Tags | Type | Description |
| ------------------------------- | ------------------------------- | --------- | -------------------- |
| jvm_buffer_memory_used_bytes | id="direct/mapped" | AutoGauge | 已经使用的缓冲区大小 |
@@ -265,12 +280,14 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| jvm_memory_used_bytes | {area="heap/nonheap",id="xxx",} | AutoGauge | 已使用内存大小 |
#### 4.2.15. JVM 类加载统计
+
| Metric | Tags | Type | Description |
| ---------------------------- | ---- | --------- | ------------------- |
| jvm_classes_unloaded_classes | | AutoGauge | 累计卸载的class数量 |
| jvm_classes_loaded_classes | | AutoGauge | 累计加载的class数量 |
-#### 4.2.16. JVM 编译时间统计
+#### 4.2.16. JVM 编译时间统计
+
| Metric | Tags | Type | Description |
| ----------------------- | --------------------------------------------- | --------- | ------------------ |
| jvm_compilation_time_ms | {compiler="HotSpot 64-Bit Tiered Compilers",} | AutoGauge | 耗费在编译上的时间 |
@@ -278,12 +295,14 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
### 4.3. Normal 级别监控指标
#### 4.3.1. 集群
+
| Metric | Tags | Type | Description |
| ------ | -------------------------------------------------------------- | --------- | ------------------------------------------------------- |
| region | name="{{DatabaseName}}",type="SchemaRegion/DataRegion" | AutoGauge | 特定节点上不同 Database 的 DataRegion/SchemaRegion 个数 |
| slot | name="{{DatabaseName}}",type="schemaSlotNumber/dataSlotNumber" | AutoGauge | 特定节点上不同 Database 的 DataSlot/SchemaSlot 个数 |
### 4.4. All 级别监控指标
+
目前还没有All级别的监控指标，后续会持续添加。
## 5. 怎样获取这些系统监控？
@@ -291,14 +310,17 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
- 监控模块的相关配置均在`conf/iotdb-{datanode/confignode}.properties`中，所有配置项支持通过`load configuration`命令热加载。
### 5.1. 使用 JMX 方式
+
对于使用 JMX 对外暴露的指标，可以通过 Jconsole 来进行查看。在进入 Jconsole 监控页面后，首先会看到 IoTDB 的各类运行情况的概览。在这里，您可以看到堆内存信息、线程信息、类信息以及服务器的 CPU 使用情况。
#### 5.1.1. 获取监控指标数据
+
连接到 JMX 后，您可以通过 "MBeans" 标签找到名为 "org.apache.iotdb.metrics" 的 "MBean"，可以在侧边栏中查看所有监控指标的具体值。
<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" alt="metric-jmx" src="https://user-images.githubusercontent.com/46039728/204018765-6fda9391-ebcf-4c80-98c5-26f34bd74df0.png">
#### 5.1.2. 获取其他相关数据
+
连接到 JMX 后，您可以通过 "MBeans" 标签找到名为 "org.apache.iotdb.service" 的 "MBean"，如下图所示，了解服务的基本状态
<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" src="https://user-images.githubusercontent.com/46039728/149951720-707f1ee8-32ee-4fde-9252-048caebd232e.png"> <br>
@@ -310,6 +332,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
### 5.2. 使用 Prometheus 方式
#### 5.2.1. 监控指标的 Prometheus 映射关系
+
> 对于 Metric Name 为 name, Tags 为 K1=V1, ..., Kn=Vn 的监控指标有如下映射，其中 value 为具体值
| 监控指标类型 | 映射关系 |
@@ -321,6 +344,7 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| Timer | name_seconds_max{k1="V1", ..., Kn="Vn"} value <br> name_seconds_sum{k1="V1", ..., Kn="Vn"} value <br> name_seconds_count{k1="V1", ..., Kn="Vn"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.0"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.25"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.5"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.75"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="1.0"} value |
#### 5.2.2. 修改配置文件
+
1) 以 DataNode 为例，修改 iotdb-datanode.properties 配置文件如下：
```properties
@@ -385,6 +409,7 @@ static_configs:
[Grafana从Prometheus查询数据并绘图的文档](https://prometheus.io/docs/visualization/grafana/#grafana-support-for-prometheus)
#### 5.2.4. Apache IoTDB Dashboard
+
我们提供了Apache IoTDB Dashboard，在Grafana中显示的效果图如下所示：
@@ -392,13 +417,16 @@ static_configs:
##### 5.2.4.1. 如何获取 Apache IoTDB Dashboard？
1. 您可以在 GitHub 上获取到Dashboard的json文件。
- 1. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-ConfigNode-Dashboard.json">Apache IoTDB ConfigNode Dashboard</a>
- 2. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-DataNode-Dashboard.json">Apache IoTDB DataNode Dashboard</a>
+ 1. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-ConfigNode-Dashboard.json">
+ Apache IoTDB ConfigNode Dashboard</a>
+ 2. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-DataNode-Dashboard.json">
+ Apache IoTDB DataNode Dashboard</a>
2. 您可以访问[Grafana Dashboard官网](https://grafana.com/grafana/dashboards/)搜索`Apache IoTDB Dashboard`并使用
在创建Grafana时，您可以选择Import刚刚下载的json文件，并为Apache IoTDB Dashboard选择对应目标数据源。
##### 5.2.4.2. Apache IoTDB ConfigNode Dashboard 说明
+
> 除特殊说明的监控项以外，以下监控项均保证在Important级别的监控框架中可用。
- `Overview`：系统概述
@@ -428,8 +456,9 @@ static_configs:
- `Log Number Per Minute`：IoTDB 进程平均每分钟日志数
- `The Time Consumed of Compliation Per Minute`：平均每分钟编译耗时
- `The Number Of Class`：JVM 加载和卸载的类数量
-
+
##### 5.2.4.3. Apache IoTDB DataNode Dashboard 说明
+
> 除特殊说明的监控项以外，以下监控项均保证在Important级别的监控框架中可用。
- `Overview`：系统概述
@@ -486,6 +515,7 @@ static_configs:
### 5.3. 使用 IoTDB 方式
#### 5.3.1. 监控指标的 IoTDB 映射关系
+
> 对于 Metric Name 为 name, Tags 为 K1=V1, ..., Kn=Vn 的监控指标有如下映射，以默认写到 root.__system.metric.`ip:port` 为例
| 监控指标类型 | 映射关系 [...]
@@ -497,4 +527,5 @@ static_configs:
| Timer | root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.count <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.max <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.mean <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.sum <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.p0 <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.p25 <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.p50 <br> root.__system.metric.`ip:por [...]
#### 5.3.2. 获取监控指标
+
根据如上的映射关系，可以构成相关的 IoTDB 查询语句获取监控指标
\ No newline at end of file
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Operation.java b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Operation.java
deleted file mode 100644
index d4a8651e5e..0000000000
--- a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Operation.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.commons.service.metric.enums;
-
-public enum Operation {
- EXECUTE_JDBC_BATCH("EXECUTE_JDBC_BATCH"),
- EXECUTE_ONE_SQL_IN_BATCH("EXECUTE_ONE_SQL_IN_BATCH"),
- EXECUTE_ROWS_PLAN_IN_BATCH("EXECUTE_ROWS_PLAN_IN_BATCH"),
- EXECUTE_MULTI_TIMESERIES_PLAN_IN_BATCH("EXECUTE_MULTI_TIMESERIES_PLAN_IN_BATCH"),
- EXECUTE_RPC_BATCH_INSERT("EXECUTE_RPC_BATCH_INSERT"),
- EXECUTE_QUERY("EXECUTE_QUERY"),
- EXECUTE_SELECT_INTO("EXECUTE_SELECT_INTO");
-
- public String getName() {
- return name;
- }
-
- String name;
-
- Operation(String name) {
- this.name = name;
- }
-}
[iotdb] 02/10: remove useless config
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 76d51bf29cef0e5aeaba64842af520632d337937
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Jan 10 10:28:53 2023 +0800
remove useless config
---
.../apache/iotdb/commons/conf/IoTDBConstant.java | 2 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 55 ----------------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 15 ------
3 files changed, 72 deletions(-)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index ca68d8283c..eb019d7cab 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -220,8 +220,6 @@ public class IoTDBConstant {
public static final String LOAD_TSFILE_FOLDER_NAME = "load";
public static final String SYNC_FOLDER_NAME = "sync";
public static final String QUERY_FOLDER_NAME = "query";
- public static final String TRACING_FOLDER_NAME = "tracing";
- public static final String TRACING_LOG = "tracing.txt";
public static final String EXT_FOLDER_NAME = "ext";
public static final String UDF_FOLDER_NAME = "udf";
public static final String TRIGGER_FOLDER_NAME = "trigger";
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 4cbff4d50c..647d6a126d 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -250,10 +250,6 @@ public class IoTDBConfig {
+ File.separator
+ IoTDBConstant.SCHEMA_FOLDER_NAME;
- /** Performance tracing directory, stores performance tracing files */
- private String tracingDir =
- IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.TRACING_FOLDER_NAME;
-
/** Query directory, stores temporary files of query */
private String queryDir =
IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.QUERY_FOLDER_NAME;
@@ -518,24 +514,9 @@ public class IoTDBConfig {
/** Whether to enable Last cache */
private boolean lastCacheEnable = true;
- /** Set true to enable statistics monitor service, false to disable statistics service. */
- private boolean enableStatMonitor = false;
-
- /** Set true to enable writing monitor time series. */
- private boolean enableMonitorSeriesWrite = false;
-
/** Cache size of {@code checkAndGetDataTypeCache}. */
private int mRemoteSchemaCacheSize = 100000;
- /** Is external sort enable. */
- private boolean enableExternalSort = true;
-
- /**
- * The threshold of items in external sort. If the number of chunks participating in sorting
- * exceeds this threshold, external sorting is enabled, otherwise memory sorting is used.
- */
- private int externalSortThreshold = 1000;
-
/** White list for sync */
private String ipWhiteList = "127.0.0.1/32";
@@ -780,9 +761,6 @@ public class IoTDBConfig {
private float udfCollectorMemoryBudgetInMB = (float) (1.0 / 3 * udfMemoryBudgetInMB);
- /** The cached record size (in MB) of each series in group by fill query */
- private float groupByFillCacheSizeInMB = (float) 1.0;
-
// time in nanosecond precision when starting up
private long startUpNanosecond = System.nanoTime();
@@ -1073,14 +1051,6 @@ public class IoTDBConfig {
this.udfMemoryBudgetInMB = udfMemoryBudgetInMB;
}
- public float getGroupByFillCacheSizeInMB() {
- return groupByFillCacheSizeInMB;
- }
-
- public void setGroupByFillCacheSizeInMB(float groupByFillCacheSizeInMB) {
- this.groupByFillCacheSizeInMB = groupByFillCacheSizeInMB;
- }
-
public float getUdfReaderMemoryBudgetInMB() {
return udfReaderMemoryBudgetInMB;
}
@@ -1156,7 +1126,6 @@ public class IoTDBConfig {
systemDir = addDataHomeDir(systemDir);
schemaDir = addDataHomeDir(schemaDir);
loadTsFileDir = addDataHomeDir(loadTsFileDir);
- tracingDir = addDataHomeDir(tracingDir);
consensusDir = addDataHomeDir(consensusDir);
dataRegionConsensusDir = addDataHomeDir(dataRegionConsensusDir);
ratisDataRegionSnapshotDir = addDataHomeDir(ratisDataRegionSnapshotDir);
@@ -1350,14 +1319,6 @@ public class IoTDBConfig {
this.schemaDir = schemaDir;
}
- public String getTracingDir() {
- return tracingDir;
- }
-
- void setTracingDir(String tracingDir) {
- this.tracingDir = tracingDir;
- }
-
public String getQueryDir() {
return queryDir;
}
@@ -1850,22 +1811,6 @@ public class IoTDBConfig {
- allocateMemoryForSchema;
}
- public boolean isEnableExternalSort() {
- return enableExternalSort;
- }
-
- void setEnableExternalSort(boolean enableExternalSort) {
- this.enableExternalSort = enableExternalSort;
- }
-
- public int getExternalSortThreshold() {
- return externalSortThreshold;
- }
-
- void setExternalSortThreshold(int externalSortThreshold) {
- this.externalSortThreshold = externalSortThreshold;
- }
-
public boolean isEnablePartialInsert() {
return enablePartialInsert;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index d0a47ede9a..d98a6be48b 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -331,8 +331,6 @@ public class IoTDBDescriptor {
conf.setQueryDir(
FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME));
- conf.setTracingDir(properties.getProperty("dn_tracing_dir", conf.getTracingDir()));
-
conf.setDataDirs(properties.getProperty("dn_data_dirs", conf.getDataDirs()[0]).split(","));
conf.setConsensusDir(properties.getProperty("dn_consensus_dir", conf.getConsensusDir()));
@@ -568,14 +566,6 @@ public class IoTDBDescriptor {
Boolean.parseBoolean(properties.getProperty("chunk_buffer_pool_enable")));
}
- conf.setEnableExternalSort(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_external_sort", Boolean.toString(conf.isEnableExternalSort()))));
- conf.setExternalSortThreshold(
- Integer.parseInt(
- properties.getProperty(
- "external_sort_threshold", Integer.toString(conf.getExternalSortThreshold()))));
conf.setUpgradeThreadCount(
Integer.parseInt(
properties.getProperty(
@@ -1694,11 +1684,6 @@ public class IoTDBDescriptor {
Math.min(Float.parseFloat(memoryBudgetInMb), 0.2 * conf.getAllocateMemoryForRead()));
}
- String groupByFillCacheSizeInMB = properties.getProperty("group_by_fill_cache_size_in_mb");
- if (groupByFillCacheSizeInMB != null) {
- conf.setGroupByFillCacheSizeInMB(Float.parseFloat(groupByFillCacheSizeInMB));
- }
-
String readerTransformerCollectorMemoryProportion =
properties.getProperty("udf_reader_transformer_collector_memory_proportion");
if (readerTransformerCollectorMemoryProportion != null) {
[iotdb] 10/10: update Metric-Tool.md (en)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 88f17c6a3b1e9583b95927ba6d8d6cea5bad9647
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Jan 12 16:21:07 2023 +0800
update Metric-Tool.md (en)
---
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 75 +++++++++++++++++++++++++++++
1 file changed, 75 insertions(+)
diff --git a/docs/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
index 6290c18fe5..aba640a3fe 100644
--- a/docs/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -317,6 +317,81 @@ carefully evaluated. The current Core-level metrics are as follows:
| ----------------------- | --------------------------------------------- | --------- | -------------------------------- |
| jvm_compilation_time_ms | {compiler="HotSpot 64-Bit Tiered Compilers",} | AutoGauge | The time consumed in compilation |
+#### 4.2.17. Query Planning
+
+| Metric | Tags | Type | Description |
+|-----------------|------------------------------|-------|-----------------------------------------------------|
+| query_plan_cost | stage="sql_parser" | Timer | The SQL parsing time-consuming |
+| query_plan_cost | stage="analyzer" | Timer | The query statement analysis time-consuming |
+| query_plan_cost | stage="logical_planner" | Timer | The query logical plan planning time-consuming |
+| query_plan_cost | stage="distribution_planner" | Timer | The query distribution plan planning time-consuming |
+| query_plan_cost | stage="partition_fetcher" | Timer | The partition information fetching time-consuming |
+| query_plan_cost | stage="schema_fetcher" | Timer | The schema information fetching time-consuming |
+
+#### 4.2.18. Plan Dispatcher
+
+| Metric | Tags | Type | Description |
+|------------|---------------------------|-------|--------------------------------------------------------------|
+| dispatcher | stage="wait_for_dispatch" | Timer | The distribution plan dispatcher time-consuming |
+| dispatcher | stage="dispatch_read" | Timer | The distribution plan dispatcher time-consuming (only query) |
+
+#### 4.2.19. Query Resource
+
+| Metric | Tags | Type | Description |
+|----------------|--------------------------|------|--------------------------------------------|
+| query_resource | type="sequence_tsfile" | Rate | The access frequency of sequence tsfiles |
+| query_resource | type="unsequence_tsfile" | Rate | The access frequency of unsequence tsfiles |
+| query_resource | type="flushing_memtable" | Rate | The access frequency of flushing memtables |
+| query_resource | type="working_memtable" | Rate | The access frequency of working memtables |
+
+#### 4.2.20. Data Exchange
+
+| Metric | Tags | Type | Description |
+|---------------------|------------------------------------------------------------------------|-----------|-----------------------------------------------------------------|
+| data_exchange_cost | operation="source_handle_get_tsblock", type="local/remote" | Timer | The time-consuming that source handles receive TsBlock |
+| data_exchange_cost | operation="source_handle_deserialize_tsblock", type="local/remote" | Timer | The time-consuming that source handles deserialize TsBlock |
+| data_exchange_cost | operation="sink_handle_send_tsblock", type="local/remote" | Timer | The time-consuming that sink handles send TsBlock |
+| data_exchange_cost | operation="send_new_data_block_event_task", type="server/caller" | Timer | The RPC time-consuming that sink handles send TsBlock |
+| data_exchange_cost | operation="get_data_block_task", type="server/caller" | Timer | The RPC time-consuming that source handles receive TsBlock |
+| data_exchange_cost | operation="on_acknowledge_data_block_event_task", type="server/caller" | Timer | The RPC time-consuming that source handles ack received TsBlock |
+| data_exchange_count | name="send_new_data_block_num", type="server/caller" | Histogram | The number of sent TsBlocks by sink handles |
+| data_exchange_count | name="get_data_block_num", type="server/caller" | Histogram | The number of received TsBlocks by source handles |
+| data_exchange_count | name="on_acknowledge_data_block_num", type="server/caller" | Histogram | The number of acknowledged TsBlocks by source handles |
+
+#### 4.2.21. Query Task Schedule
+
+| Metric | Tags | Type | Description |
+|------------------|--------------------------------|-----------|--------------------------------------------------|
+| driver_scheduler | name="ready_queued_time" | Timer | The queuing time of ready queue |
+| driver_scheduler | name="block_queued_time" | Timer | The queuing time of blocking queue |
+| driver_scheduler | name="ready_queue_task_count" | AutoGauge | The number of tasks queued in the ready queue |
+| driver_scheduler | name="block_queued_task_count" | AutoGauge | The number of tasks queued in the blocking queue |
+
+#### 4.2.22. Query Execution
+
+| Metric | Tags | Type | Description |
+|--------------------------|-------------------------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------|
+| query_execution | stage="local_execution_planner" | Timer | The time-consuming of operator tree construction |
+| query_execution | stage="query_resource_init" | Timer | The time-consuming of query resource initialization |
+| query_execution | stage="get_query_resource_from_mem" | Timer | The time-consuming of query resource memory query and construction |
+| query_execution | stage="driver_internal_process" | Timer | The time-consuming of driver execution |
+| query_execution | stage="wait_for_result" | Timer | The time-consuming of getting query result from result handle |
+| operator_execution_cost | name="{{operator_name}}" | Timer | The operator execution time |
+| operator_execution_count | name="{{operator_name}}" | Counter | The number of operator calls (counted by the number of next method calls) |
+| aggregation | from="raw_data" | Timer | The time-consuming of performing an aggregation calculation from a batch of raw data |
+| aggregation | from="statistics" | Timer | The time-consuming of updating an aggregated value with statistics |
+| series_scan_cost | stage="load_timeseries_metadata", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of loading TimeseriesMetadata |
+| series_scan_cost | stage="read_timeseries_metadata", type="", from="cache/file" | Timer | The time-consuming of reading TimeseriesMetadata of a tsfile |
+| series_scan_cost | stage="timeseries_metadata_modification", type="aligned/non_aligned", from="null" | Timer | The time-consuming of filtering TimeseriesMetadata by mods |
+| series_scan_cost | stage="load_chunk_metadata_list", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of loading ChunkMetadata list |
+| series_scan_cost | stage="chunk_metadata_modification", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of filtering ChunkMetadata by mods |
+| series_scan_cost | stage="chunk_metadata_filter", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of filtering ChunkMetadata by query filter |
+| series_scan_cost | stage="construct_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of constructing ChunkReader |
+| series_scan_cost | stage="read_chunk", type="", from="cache/file" | Timer | The time-consuming of reading Chunk |
+| series_scan_cost | stage="init_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of initializing ChunkReader (constructing PageReader) |
+| series_scan_cost | stage="build_tsblock_from_page_reader", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of constructing Tsblock from PageReader |
+| series_scan_cost | stage="build_tsblock_from_merge_reader", type="aligned/non_aligned", from="null" | Timer | The time-consuming of constructing Tsblock from MergeReader (handling overlapping data) |
+
### 4.3. Normal level Metrics
#### 4.3.1. Cluster
[iotdb] 01/10: remove SubRawQuery
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8319a0b5b5aeef7e013ca8cc43ca710f8b092ba6
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Jan 10 10:08:08 2023 +0800
remove SubRawQuery
---
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 1 -
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 1 -
.../iotdb/commons/concurrent/ThreadName.java | 1 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 24 ----------------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 15 --------------
5 files changed, 42 deletions(-)
diff --git a/docs/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
index 1bce86a4bc..0f5aec4249 100644
--- a/docs/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -196,7 +196,6 @@ Core-level metrics are enabled by default during system operation. The addition
| queue | name="compaction_cross", status="running/waiting" | Gauge | The number of cross compatcion tasks |
| cost_task | name="inner_compaction/cross_compaction/flush" | Gauge | The time consumed of compaction tasks |
| queue | name="flush",status="running/waiting" | AutoGauge | The number of flush tasks |
-| queue | name="Sub_RawQuery",status="running/waiting" | AutoGauge | The number of Sub_RawQuery |
#### 4.2.8. Compaction
| Metric | Tags | Type | Description |
diff --git a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
index 92f3d0e32d..b32ade409f 100644
--- a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -192,7 +192,6 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| queue | name="compaction_cross", status="running/waiting" | Gauge | 跨空间合并任务数 |
| cost_task | name="inner_compaction/cross_compaction/flush" | Gauge | 任务耗时情况 |
| queue | name="flush",status="running/waiting" | AutoGauge | 刷盘任务数 |
-| queue | name="Sub_RawQuery",status="running/waiting" | AutoGauge | Sub_RawQuery任务数 |
#### 4.2.8. 合并统计
| Metric | Tags | Type | Description |
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
index f0d3bac43a..9cdf23ae5a 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
@@ -45,7 +45,6 @@ public enum ThreadName {
SYNC_CLIENT("Sync-Client"),
SYNC_SERVER("Sync"),
QUERY_SERVICE("Query"),
- SUB_RAW_QUERY_SERVICE("Sub_RawQuery"),
INSERTION_SERVICE("MultithreadingInsertionPool"),
WINDOW_EVALUATION_SERVICE("WindowEvaluationTaskPoolManager"),
TTL_CHECK_SERVICE("TTL-CHECK"),
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 0268ff12a5..4cbff4d50c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -324,14 +324,6 @@ public class IoTDBConfig {
/** How many queries can be concurrently executed. When <= 0, use 1000. */
private int maxAllowedConcurrentQueries = 1000;
- /**
- * How many threads can concurrently read data for raw data query. When <= 0, use CPU core number.
- */
- private int subRawQueryThreadCount = 8;
-
- /** Blocking queue size for read task in raw data query. */
- private int rawQueryBlockingQueueCapacity = 5;
-
/** How many threads can concurrently evaluate windows. When <= 0, use CPU core number. */
private int windowEvaluationThreadCount = Runtime.getRuntime().availableProcessors();
@@ -1520,26 +1512,10 @@ public class IoTDBConfig {
this.maxAllowedConcurrentQueries = maxAllowedConcurrentQueries;
}
- public int getSubRawQueryThreadCount() {
- return subRawQueryThreadCount;
- }
-
- void setSubRawQueryThreadCount(int subRawQueryThreadCount) {
- this.subRawQueryThreadCount = subRawQueryThreadCount;
- }
-
public long getMaxBytesPerFragmentInstance() {
return allocateMemoryForDataExchange / queryThreadCount;
}
- public int getRawQueryBlockingQueueCapacity() {
- return rawQueryBlockingQueueCapacity;
- }
-
- public void setRawQueryBlockingQueueCapacity(int rawQueryBlockingQueueCapacity) {
- this.rawQueryBlockingQueueCapacity = rawQueryBlockingQueueCapacity;
- }
-
public int getWindowEvaluationThreadCount() {
return windowEvaluationThreadCount;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index fd6513a688..d0a47ede9a 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -553,21 +553,6 @@ public class IoTDBDescriptor {
conf.setMaxAllowedConcurrentQueries(1000);
}
- conf.setSubRawQueryThreadCount(
- Integer.parseInt(
- properties.getProperty(
- "sub_rawQuery_thread_count", Integer.toString(conf.getSubRawQueryThreadCount()))));
-
- if (conf.getSubRawQueryThreadCount() <= 0) {
- conf.setSubRawQueryThreadCount(Runtime.getRuntime().availableProcessors());
- }
-
- conf.setRawQueryBlockingQueueCapacity(
- Integer.parseInt(
- properties.getProperty(
- "raw_query_blocking_queue_capacity",
- Integer.toString(conf.getRawQueryBlockingQueueCapacity()))));
-
conf.setmRemoteSchemaCacheSize(
Integer.parseInt(
properties
[iotdb] 04/10: update metric name (operation -> statement_execution)(en)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6f578dd78b2317e4de9d6a58adcf9b249020fc8d
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Jan 10 10:49:41 2023 +0800
update metric name (operation -> statement_execution)(en)
---
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 117 ++++++++++++++++++++--------
1 file changed, 86 insertions(+), 31 deletions(-)
diff --git a/docs/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
index 0f5aec4249..6290c18fe5 100644
--- a/docs/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -51,7 +51,9 @@ Belows are some typical application scenarios
running abnormally.
## 2. Who will use metric framework?
-Any person cares about the system's status, including but not limited to RD, QA, SRE, DBA, can use the metrics to work more efficiently.
+
+Any person cares about the system's status, including but not limited to RD, QA, SRE, DBA, can use the metrics to work
+more efficiently.
## 3. What is metrics?
@@ -60,19 +62,36 @@ Any person cares about the system's status, including but not limited to RD, QA,
In IoTDB's metric module, each metrics is uniquely identified by `Metric Name` and `Tags`.
- `Metric Name`: Metric type name, such as `logback_events` means log events.
-- `Tags`: indicator classification, in the form of Key-Value pairs, each indicator can have 0 or more categories, common Key-Value pairs:
- - `name = xxx`: The name of the monitored object, which is the description of **business logic**. For example, for a monitoring item of type `Metric Name = entry_seconds_count`, the meaning of name refers to the monitored business interface.
- - `type = xxx`: Monitoring indicator type subdivision, which is a description of **monitoring indicator** itself. For example, for monitoring items of type `Metric Name = point`, the meaning of type refers to the specific type of monitoring points.
- - `status = xxx`: The status of the monitored object is a description of **business logic**. For example, for monitoring items of type `Metric Name = Task`, this parameter can be used to distinguish the status of the monitored object.
- - `user = xxx`: The relevant user of the monitored object is a description of **business logic**. For example, count the total points written by the `root` user.
- - Customize according to the specific situation: For example, there is a level classification under logback_events_total, which is used to indicate the number of logs under a specific level.
-- `Metric Level`: The level of metric managing level, The default startup level is `Core` level, the recommended startup level is `Important level`, and the audit strictness is `Core > Important > Normal > All`
- - `Core`: Core metrics of the system, used by the **operation and maintenance personnel**, which is related to the **performance, stability, and security** of the system, such as the status of the instance, the load of the system, etc.
- - `Important`: Important metrics of the module, which is used by **operation and maintenance and testers**, and is directly related to **the running status of each module**, such as the number of merged files, execution status, etc.
- - `Normal`: Normal metrics of the module, used by **developers** to facilitate **locating the module** when problems occur, such as specific key operation situations in the merger.
- - `All`: All metrics of the module, used by **module developers**, often used when the problem is reproduced, so as to solve the problem quickly.
+- `Tags`: indicator classification, in the form of Key-Value pairs, each indicator can have 0 or more categories, common
+ Key-Value pairs:
+ - `name = xxx`: The name of the monitored object, which is the description of **business logic**. For example, for a
+ monitoring item of type `Metric Name = entry_seconds_count`, the meaning of name refers to the monitored business
+ interface.
+ - `type = xxx`: Monitoring indicator type subdivision, which is a description of **monitoring indicator** itself.
+ For example, for monitoring items of type `Metric Name = point`, the meaning of type refers to the specific type
+ of monitoring points.
+ - `status = xxx`: The status of the monitored object is a description of **business logic**. For example, for
+ monitoring items of type `Metric Name = Task`, this parameter can be used to distinguish the status of the
+ monitored object.
+ - `user = xxx`: The relevant user of the monitored object is a description of **business logic**. For example, count
+ the total points written by the `root` user.
+ - Customize according to the specific situation: For example, there is a level classification under
+ logback_events_total, which is used to indicate the number of logs under a specific level.
+- `Metric Level`: The level of metric managing level, The default startup level is `Core` level, the recommended startup
+ level is `Important level`, and the audit strictness is `Core > Important > Normal > All`
+ - `Core`: Core metrics of the system, used by the **operation and maintenance personnel**, which is related to
+ the **performance, stability, and security** of the system, such as the status of the instance, the load of the
+ system, etc.
+ - `Important`: Important metrics of the module, which is used by **operation and maintenance and testers**, and is
+ directly related to **the running status of each module**, such as the number of merged files, execution status,
+ etc.
+ - `Normal`: Normal metrics of the module, used by **developers** to facilitate **locating the module** when problems
+ occur, such as specific key operation situations in the merger.
+ - `All`: All metrics of the module, used by **module developers**, often used when the problem is reproduced, so as
+ to solve the problem quickly.
### 3.2. External data format for metrics
+
- IoTDB provides metrics in JMX, Prometheus and IoTDB formats:
- For JMX, metrics can be obtained through ```org.apache.iotdb.metrics```.
- For Prometheus, the value of the metrics can be obtained through the externally exposed port
@@ -87,9 +106,12 @@ If you want to add your own metrics data in IoTDB, please see
the [IoTDB Metric Framework] (https://github.com/apache/iotdb/tree/master/metrics) document.
### 4.1. Core level metrics
-Core-level metrics are enabled by default during system operation. The addition of each Core-level metrics needs to be carefully evaluated. The current Core-level metrics are as follows:
+
+Core-level metrics are enabled by default during system operation. The addition of each Core-level metrics needs to be
+carefully evaluated. The current Core-level metrics are as follows:
#### 4.1.1. Cluster
+
| Metric | Tags | Type | Description |
| ----------- | ----------------------------------------------- | --------- | --------------------------------------------------- |
| config_node | name="total",status="Registered/Online/Unknown" | AutoGauge | The number of registered/online/unknown confignodes |
@@ -97,6 +119,7 @@ Core-level metrics are enabled by default during system operation. The addition
| points | database="{{database}}", type="flush" | Gauge | The point number of last flushed memtable |
#### 4.1.2. IoTDB process
+
| Metric | Tags | Type | Description |
| ----------------- | -------------- | --------- | ------------------------------------------------------ |
| process_cpu_load | name="process" | AutoGauge | The current CPU usage of IoTDB process, Unit: % |
@@ -106,6 +129,7 @@ Core-level metrics are enabled by default during system operation. The addition
| process_free_mem | name="memory" | AutoGauge | The free available memory of IoTDB process |
#### 4.1.3. System
+
| Metric | Tags | Type | Description |
| ------------------------------ | ------------- | --------- | ---------------------------------------------------------- |
| sys_cpu_load | name="system" | AutoGauge | The current CPU usage of system, Unit: % |
@@ -121,12 +145,14 @@ Core-level metrics are enabled by default during system operation. The addition
### 4.2. Important level metrics
#### 4.2.1. Cluster
+
| Metric | Tags | Type | Description |
| ------------------------- | ------------------------------------------------- | ----- | ------------------------------------------------ |
| cluster_node_leader_count | name="{{ip}}:{{port}}" | Gauge | The count of consensus group leader on each node |
| cluster_node_status | name="{{ip}}:{{port}}",type="ConfigNode/DataNode" | Gauge | The current node status, 0=Unkonwn 1=online |
#### 4.2.2. Node
+
| Metric | Tags | Type | Description |
| -------- | ------------------------------------------ | --------- | ------------------------------------------------------------- |
| quantity | name="database" | AutoGauge | The number of database |
@@ -138,6 +164,7 @@ Core-level metrics are enabled by default during system operation. The addition
| region | name="{{ip}}:{{port}}",type="DataRegion" | Gauge | The number of DataRegion in PartitionTable of specific node |
#### 4.2.3. IoTConsensus
+
| Metric | Tags | Type | Description |
| ------------ | -------------------------------------------------------------------------------------------- | --------- | --------------------------------------------------------------------- |
| mutli_leader | name="logDispatcher-{{IP}}:{{Port}}", region="{{region}}", type="currentSyncIndex" | AutoGauge | The sync index of synchronization thread in replica group |
@@ -153,6 +180,7 @@ Core-level metrics are enabled by default during system operation. The addition
| stage | name="iot_consensus", region="{{region}}", type="syncLogTimePerRequest" | Histogram | The time consumed to sync log in asynchronous callback process |
#### 4.2.4. Cache
+
| Metric | Tags | Type | Description |
| --------- | ---------------------------------- | --------- | ------------------------------------------------------------------------ |
| cache_hit | name="chunk" | AutoGauge | The cache hit ratio of ChunkCache, Unit: % |
@@ -167,20 +195,22 @@ Core-level metrics are enabled by default during system operation. The addition
| cache | name="DataPartition", type="all" | Counter | The access number of SDataPartition Cache |
#### 4.2.5. Interface
-| Metric | Tags | Type | Description |
-| --------------------- | ---------------------------------- | --------- | -------------------------------------------------------------- |
-| operation | name = "{{name}}" | Histogram | The time consumed of operations in client |
-| entry | name="{{interface}}" | Timer | The time consumed of thrift operations |
-| thrift_connections | name="ConfigNodeRPC" | AutoGauge | The number of thrift internal connections in ConfigNode |
-| thrift_connections | name="Internal" | AutoGauge | The number of thrift internal connections in DataNode |
-| thrift_connections | name="MPPDataExchange" | AutoGauge | The number of thrift internal connections in MPP |
-| thrift_connections | name="RPC" | AutoGauge | The number of thrift connections of Client |
-| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | The number of thrift active internal connections in ConfigNode |
-| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | The number of thrift active internal connections in DataNode |
-| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | The number of thrift active internal connections in MPP |
-| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | The number of thrift active connections of client |
+
+| Metric | Tags | Type | Description |
+| --------------------- | ---------------------------------------------------- | --------- | -------------------------------------------------------------- |
+| statement_execution | interface="{{interface}}", type="{{statement_type}}" | Timer | The time consumed of operations in client |
+| entry | name="{{interface}}" | Timer | The time consumed of thrift operations |
+| thrift_connections | name="ConfigNodeRPC" | AutoGauge | The number of thrift internal connections in ConfigNode |
+| thrift_connections | name="Internal" | AutoGauge | The number of thrift internal connections in DataNode |
+| thrift_connections | name="MPPDataExchange" | AutoGauge | The number of thrift internal connections in MPP |
+| thrift_connections | name="RPC" | AutoGauge | The number of thrift connections of Client |
+| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | The number of thrift active internal connections in ConfigNode |
+| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | The number of thrift active internal connections in DataNode |
+| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | The number of thrift active internal connections in MPP |
+| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | The number of thrift active connections of client |
#### 4.2.6. Memory
+
| Metric | Tags | Type | Description |
| ------ | ------------------------------------ | --------- | ------------------------------------------------------------------ |
| mem | name="database_{{name}}" | AutoGauge | The memory usage of DataRegion in DataNode, Unit: byte |
@@ -190,6 +220,7 @@ Core-level metrics are enabled by default during system operation. The addition
| mem | name="schema_region_total_remaining" | AutoGauge | The memory remaining for all SchemaRegion, Unit: byte |
#### 4.2.7. Task
+
| Metric | Tags | Type | Description |
| --------- | ------------------------------------------------- | --------- | ------------------------------------- |
| queue | name="compaction_inner", status="running/waiting" | Gauge | The number of inner compaction tasks |
@@ -198,6 +229,7 @@ Core-level metrics are enabled by default during system operation. The addition
| queue | name="flush",status="running/waiting" | AutoGauge | The number of flush tasks |
#### 4.2.8. Compaction
+
| Metric | Tags | Type | Description |
| --------------------- | --------------------------------------------------- | ------- | -------------------------------------- |
| data_written | name="compaction", type="aligned/not-aligned/total" | Counter | The written size of compaction |
@@ -234,6 +266,7 @@ Core-level metrics are enabled by default during system operation. The addition
| process_status | name="process" | AutoGauge | The status of IoTDB process, 1=live, 0=dead |
#### 4.2.11. Log
+
| Metric | Tags | Type | Description |
| -------------- | ----------------------------------- | ------- | ------------------------ |
| logback_events | level="trace/debug/info/warn/error" | Counter | The number of log events |
@@ -248,6 +281,7 @@ Core-level metrics are enabled by default during system operation. The addition
| jvm_threads_states_threads | state="runnable/blocked/waiting/timed-waiting/new/terminated" | AutoGauge | The number of thread in different states |
#### 4.2.13. JVM GC
+
| Metric | Tags | Type | Description |
| ----------------------------- | ----------------------------------------------------- | --------- | --------------------------------------------------------------------------- |
| jvm_gc_pause | action="end of major GC/end of minor GC",cause="xxxx" | Timer | The number and time consumed of Young GC/Full Gc caused by different reason |
@@ -260,6 +294,7 @@ Core-level metrics are enabled by default during system operation. The addition
| jvm_gc_memory_allocated_bytes | | Counter | The accumulative value of positive memory growth of allocated memory |
#### 4.2.14. JVM Memory
+
| Metric | Tags | Type | Description |
| ------------------------------- | ------------------------------- | --------- | --------------------------- |
| jvm_buffer_memory_used_bytes | id="direct/mapped" | AutoGauge | The used size of buffer |
@@ -270,12 +305,14 @@ Core-level metrics are enabled by default during system operation. The addition
| jvm_memory_used_bytes | {area="heap/nonheap",id="xxx",} | AutoGauge | The used memory of JVM |
#### 4.2.15. JVM Class
+
| Metric | Tags | Type | Description |
| ---------------------------- | ---- | --------- | ---------------------------- |
| jvm_classes_unloaded_classes | | AutoGauge | The number of unloaded class |
| jvm_classes_loaded_classes | | AutoGauge | The number of loaded class |
#### 4.2.16. JVM Compilation
+
| Metric | Tags | Type | Description |
| ----------------------- | --------------------------------------------- | --------- | -------------------------------- |
| jvm_compilation_time_ms | {compiler="HotSpot 64-Bit Tiered Compilers",} | AutoGauge | The time consumed in compilation |
@@ -283,12 +320,14 @@ Core-level metrics are enabled by default during system operation. The addition
### 4.3. Normal level Metrics
#### 4.3.1. Cluster
+
| Metric | Tags | Type | Description |
| ------ | -------------------------------------------------------------- | --------- | ------------------------------------------------------------------ |
| region | name="{{DatabaseName}}",type="SchemaRegion/DataRegion" | AutoGauge | The number of DataRegion/SchemaRegion of database in specific node |
| slot | name="{{DatabaseName}}",type="schemaSlotNumber/dataSlotNumber" | AutoGauge | The number of DataSlot/SchemaSlot of database in specific node |
### 4.4. All Metric
+
Currently there is no All level metrics, and it will continue to be added in the future.
## 5. How to get these metrics？
@@ -297,15 +336,22 @@ The relevant configuration of the metric module is in `conf/iotdb-{datanode/conf
configuration items support hot loading through the `load configuration` command.
### 5.1. JMX
-For metrics exposed externally using JMX, you can view them through Jconsole. After entering the Jconsole monitoring page, you will first see an overview of various running conditions of IoTDB. Here you can see heap memory information, thread information, class information, and the server's CPU usage.
+
+For metrics exposed externally using JMX, you can view them through Jconsole. After entering the Jconsole monitoring
+page, you will first see an overview of various running conditions of IoTDB. Here you can see heap memory information,
+thread information, class information, and the server's CPU usage.
#### 5.1.1. Obtain metric data
-After connecting to JMX, you can find the "MBean" named "org.apache.iotdb.metrics" through the "MBeans" tab, and you can view the specific values of all monitoring metrics in the sidebar.
+
+After connecting to JMX, you can find the "MBean" named "org.apache.iotdb.metrics" through the "MBeans" tab, and you can
+view the specific values of all monitoring metrics in the sidebar.
<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" alt="metric-jmx" src="https://user-images.githubusercontent.com/46039728/204018765-6fda9391-ebcf-4c80-98c5-26f34bd74df0.png">
#### 5.1.2. Get other relevant data
-After connecting to JMX, you can find the "MBean" named "org.apache.iotdb.service" through the "MBeans" tab, as shown in the image below, to understand the basic status of the service
+
+After connecting to JMX, you can find the "MBean" named "org.apache.iotdb.service" through the "MBeans" tab, as shown in
+the image below, to understand the basic status of the service
<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" src="https://user-images.githubusercontent.com/46039728/149951720-707f1ee8-32ee-4fde-9252-048caebd232e.png"> <br>
@@ -317,6 +363,7 @@ the sidebar `org.apache.iotdb.db.service` to view the cache hit ratio:
### 5.2. Prometheus
#### 5.2.1. The mapping from metric type to prometheus format
+
> For metrics whose Metric Name is name and Tags are K1=V1, ..., Kn=Vn, the mapping is as follows, where value is a specific value
| Metric Type | Mapping |
@@ -328,6 +375,7 @@ the sidebar `org.apache.iotdb.db.service` to view the cache hit ratio:
| Timer | name_seconds_max{k1="V1", ..., Kn="Vn"} value <br> name_seconds_sum{k1="V1", ..., Kn="Vn"} value <br> name_seconds_count{k1="V1", ..., Kn="Vn"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.0"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.25"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.5"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="0.75"} value <br> name_seconds{k1="V1", ..., Kn="Vn", quantile="1.0"} value |
#### 5.2.2. Config File
+
1) Taking DataNode as an example, modify the iotdb-datanode.properties configuration file as follows:
```properties
@@ -396,6 +444,7 @@ The following documents may help you have a good journey with Prometheus and Gra
[Grafana query metrics from Prometheus](https://prometheus.io/docs/visualization/grafana/#grafana-support-for-prometheus)
#### 5.2.4. Apache IoTDB Dashboard
+
We provide the Apache IoTDB Dashboard, and the rendering shown in Grafana is as follows:
@@ -403,8 +452,10 @@ We provide the Apache IoTDB Dashboard, and the rendering shown in Grafana is as
##### 5.2.4.1. How to get Apache IoTDB Dashboard
1. You can obtain the json files of Dashboards by GitHub:
- 1. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-ConfigNode-Dashboard.json">Apache IoTDB ConfigNode Dashboard</a>
- 2. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-DataNode-Dashboard.json">Apache IoTDB DataNode Dashboard</a>
+ 1. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-ConfigNode-Dashboard.json">
+ Apache IoTDB ConfigNode Dashboard</a>
+ 2. <a href = "https://github.com/apache/iotdb/tree/master/docs/UserGuide/Monitor-Alert/Apache-IoTDB-DataNode-Dashboard.json">
+ Apache IoTDB DataNode Dashboard</a>
2. You can visit [Grafana Dashboard official website](https://grafana.com/grafana/dashboards/), search
for `Apache IoTDB Dashboard` and use
@@ -412,6 +463,7 @@ When creating Grafana, you can select the json file you just downloaded to `Impo
data source for Apache IoTDB Dashboard.
##### 5.2.4.2. Apache IoTDB ConfigNode Dashboard Instructions
+
> Except for the metrics specified specially, the following metrics are guaranteed to be available in the monitoring framework at the Important levels.
- `Overview`: system overview
@@ -448,8 +500,9 @@ data source for Apache IoTDB Dashboard.
- `Log Number Per Minute`: the average number of logs per minute of the IoTDB process
- `The Time Consumed of Compliation Per Minute`: average compilation time per minute
- `The Number Of Class`: The number of classes loaded and unloaded by the JVM
-
+
##### 5.2.4.3. Apache IoTDB DataNode Dashboard Instructions
+
> Except for the metrics specified specially, the following metrics are guaranteed to be available in the monitoring framework at the Important levels.
- `Overview`: system overview
@@ -513,6 +566,7 @@ data source for Apache IoTDB Dashboard.
### 5.3. IoTDB
#### 5.3.1. IoTDB mapping relationship of metrics
+
> For metrics whose Metric Name is name and Tags are K1=V1, ..., Kn=Vn, the mapping is as follows, taking root.__system.metric.`ip:port` as an example by default
| Metric Type | Mapping [...]
@@ -524,4 +578,5 @@ data source for Apache IoTDB Dashboard.
| Timer | root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.count <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.max <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.mean <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.sum <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.p0 <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.p25 <br> root.__system.metric.`ip:port`.name.`K1=V1`...`Kn=Vn`.p50 <br> root.__system.metric.`ip:por [...]
#### 5.3.2. Obtain metrics
+
According to the above mapping relationship, related IoTDB query statements can be formed to obtain metrics
\ No newline at end of file
[iotdb] 05/10: Merge remote-tracking branch 'origin/master' into lmh/addMetricDoc
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3b7e26c8e891a7ef8b2e944a50ede853746c3b01
Merge: 6f578dd78b dfb013b710
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Jan 10 14:29:23 2023 +0800
Merge remote-tracking branch 'origin/master' into lmh/addMetricDoc
.../iotdb/confignode/conf/ConfigNodeConstant.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 2 +-
.../procedure/env/DataNodeRemoveHandler.java | 15 +-
.../impl/statemachine/RegionMigrateProcedure.java | 15 +-
docs/UserGuide/Reference/Status-Codes.md | 15 +-
docs/zh/UserGuide/Reference/Status-Codes.md | 249 ++++++++---------
.../apache/iotdb/it/env/cluster/AbstractEnv.java | 15 +-
.../iotdb/it/env/cluster/AbstractNodeWrapper.java | 36 ++-
.../iotdb/it/env/cluster/ConfigNodeWrapper.java | 19 +-
.../iotdb/it/env/cluster/DataNodeWrapper.java | 9 +
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 25 +-
.../iotdb/it/env/cluster/MppClusterConfig.java | 15 ++
.../apache/iotdb/it/env/cluster/MppJVMConfig.java | 116 ++++++++
.../iotdb/it/env/remote/RemoteClusterConfig.java | 12 +
.../iotdb/it/env/remote/RemoteJVMConfig.java | 30 +--
.../org/apache/iotdb/itbase/env/ClusterConfig.java | 30 ++-
.../org/apache/iotdb/itbase/env/JVMConfig.java | 41 +--
.../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 6 +-
.../it/cluster/IoTDBClusterRestartIT.java | 3 +-
.../iotdb/db/integration/IoTDBCheckConfigIT.java | 3 -
.../apache/iotdb/commons/utils/StatusUtils.java | 4 +-
.../iotdb/db/localconfignode/LocalConfigNode.java | 294 ---------------------
.../db/localconfignode/LocalDataPartitionInfo.java | 111 --------
.../localconfignode/LocalDataPartitionTable.java | 127 ---------
.../localconfignode/LocalSchemaPartitionTable.java | 122 ---------
.../iotdb/db/metadata/LocalSchemaProcessor.java | 250 ------------------
.../iotdb/db/metadata/mtree/ConfigMTree.java | 21 --
.../db/metadata/schemaregion/SchemaEngine.java | 3 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 5 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 13 +-
.../storagegroup/IStorageGroupSchemaManager.java | 53 ----
.../storagegroup/StorageGroupSchemaManager.java | 78 ------
.../schema/NodePathsSchemaScanOperator.java | 159 -----------
.../config/executor/ClusterConfigTaskExecutor.java | 81 +++---
.../db/mpp/plan/scheduler/ClusterScheduler.java | 2 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 2 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 4 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 4 +-
.../iotdb/db/engine/MetadataManagerHelper.java | 158 -----------
.../compaction/TestUtilsForAlignedSeries.java | 42 +--
.../db/engine/compaction/cross/MergeTest.java | 19 --
.../inner/AbstractInnerSpaceCompactionTest.java | 19 --
.../inner/FastCompactionPerformerAlignedTest.java | 3 -
...nerSeqCompactionWithReadChunkPerformerTest.java | 17 --
...rUnseqCompactionWithReadPointPerformerTest.java | 19 --
.../ReadChunkCompactionPerformerAlignedTest.java | 3 -
.../inner/sizetiered/SizeTieredCompactionTest.java | 19 --
.../recover/SizeTieredCompactionRecoverTest.java | 16 --
.../db/engine/memtable/MemTableFlushTaskTest.java | 2 -
.../iotdb/db/metadata/mtree/ConfigMTreeTest.java | 6 -
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 12 -
.../execution/operator/AlignedSeriesTestUtil.java | 53 ----
.../iotdb/db/rescon/ResourceManagerTest.java | 19 --
.../DataNodeInternalRPCServiceImplTest.java | 11 +-
.../db/sync/datasource/PipeOpManagerTest.java | 3 -
.../apache/iotdb/db/utils/EnvironmentUtils.java | 4 -
.../org/apache/iotdb/db/utils/SchemaTestUtils.java | 22 +-
.../db/wal/recover/WALRecoverManagerTest.java | 32 ---
.../db/wal/recover/file/TsFilePlanRedoerTest.java | 53 ----
.../file/UnsealedTsFileRecoverPerformerTest.java | 31 ---
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../apache/iotdb/spark/db/EnvironmentUtils.java | 3 -
62 files changed, 499 insertions(+), 2058 deletions(-)
[iotdb] 09/10: update Metric-Tool.md (zh)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 398bd8280b23022ffee7a5da4f88686fdceaff42
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Jan 12 15:43:53 2023 +0800
update Metric-Tool.md (zh)
---
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 75 ++++++++++++++++++++++++++
1 file changed, 75 insertions(+)
diff --git a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
index 2af7c8756e..eb1a76920c 100644
--- a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -292,6 +292,81 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| ----------------------- | --------------------------------------------- | --------- | ------------------ |
| jvm_compilation_time_ms | {compiler="HotSpot 64-Bit Tiered Compilers",} | AutoGauge | 耗费在编译上的时间 |
+#### 4.2.17. 查询规划耗时统计
+
+| Metric | Tags | Type | Description |
+|-----------------|------------------------------|-------|----------------------------|
+| query_plan_cost | stage="sql_parser" | Timer | SQL 解析耗时 |
+| query_plan_cost | stage="analyzer" | Timer | 查询语句分析耗时 |
+| query_plan_cost | stage="logical_planner" | Timer | 查询逻辑计划规划耗时 |
+| query_plan_cost | stage="distribution_planner" | Timer | 查询分布式执行计划规划耗时 |
+| query_plan_cost | stage="partition_fetcher" | Timer | 分区信息拉取耗时 |
+| query_plan_cost | stage="schema_fetcher" | Timer | 元数据信息拉取耗时 |
+
+#### 4.2.18. 执行计划分发耗时统计
+
+| Metric | Tags | Type | Description |
+|------------|---------------------------|-------|----------------------|
+| dispatcher | stage="wait_for_dispatch" | Timer | 分发执行计划耗时 |
+| dispatcher | stage="dispatch_read" | Timer | 查询执行计划发送耗时 |
+
+#### 4.2.19. 查询资源访问统计
+
+| Metric | Tags | Type | Description |
+|----------------|--------------------------|------|----------------------------|
+| query_resource | type="sequence_tsfile" | Rate | 顺序文件访问频率 |
+| query_resource | type="unsequence_tsfile" | Rate | 乱序文件访问频率 |
+| query_resource | type="flushing_memtable" | Rate | flushing memtable 访问频率 |
+| query_resource | type="working_memtable" | Rate | working memtable 访问频率 |
+
+#### 4.2.20. 数据传输模块统计
+
+| Metric | Tags | Type | Description |
+|---------------------|------------------------------------------------------------------------|-----------|-----------------------------------------|
+| data_exchange_cost | operation="source_handle_get_tsblock", type="local/remote" | Timer | source handle 接收 TsBlock 耗时 |
+| data_exchange_cost | operation="source_handle_deserialize_tsblock", type="local/remote" | Timer | source handle 反序列化 TsBlock 耗时 |
+| data_exchange_cost | operation="sink_handle_send_tsblock", type="local/remote" | Timer | sink handle 发送 TsBlock 耗时 |
+| data_exchange_cost | operation="send_new_data_block_event_task", type="server/caller" | Timer | sink handle 发送 TsBlock RPC 耗时 |
+| data_exchange_cost | operation="get_data_block_task", type="server/caller" | Timer | source handle 接收 TsBlock RPC 耗时 |
+| data_exchange_cost | operation="on_acknowledge_data_block_event_task", type="server/caller" | Timer | source handle 确认接收 TsBlock RPC 耗时 |
+| data_exchange_count | name="send_new_data_block_num", type="server/caller" | Histogram | sink handle 发送 TsBlock数量 |
+| data_exchange_count | name="get_data_block_num", type="server/caller" | Histogram | source handle 接收 TsBlock 数量 |
+| data_exchange_count | name="on_acknowledge_data_block_num", type="server/caller" | Histogram | source handle 确认接收 TsBlock 数量 |
+
+#### 4.2.21. 查询任务调度统计
+
+| Metric | Tags | Type | Description |
+|------------------|--------------------------------|-----------|--------------------|
+| driver_scheduler | name="ready_queued_time" | Timer | 就绪队列排队时间 |
+| driver_scheduler | name="block_queued_time" | Timer | 阻塞队列排队时间 |
+| driver_scheduler | name="ready_queue_task_count" | AutoGauge | 就绪队列排队任务数 |
+| driver_scheduler | name="block_queued_task_count" | AutoGauge | 阻塞队列排队任务数 |
+
+#### 4.2.22. 查询执行耗时统计
+
+| Metric | Tags | Type | Description |
+|--------------------------|-------------------------------------------------------------------------------------|---------|------------------------------------------------|
+| query_execution | stage="local_execution_planner" | Timer | 算子树构造耗时 |
+| query_execution | stage="query_resource_init" | Timer | 查询资源初始化耗时 |
+| query_execution | stage="get_query_resource_from_mem" | Timer | 查询资源内存查询与构造耗时 |
+| query_execution | stage="driver_internal_process" | Timer | Driver 执行耗时 |
+| query_execution | stage="wait_for_result" | Timer | 从resultHandle 获取一次查询结果的耗时 |
+| operator_execution_cost | name="{{operator_name}}" | Timer | 算子执行耗时 |
+| operator_execution_count | name="{{operator_name}}" | Counter | 算子调用次数（以 next 方法调用次数计算） |
+| aggregation | from="raw_data" | Timer | 从一批原始数据进行一次聚合计算的耗时 |
+| aggregation | from="statistics" | Timer | 使用统计信息更新一次聚合值的耗时 |
+| series_scan_cost | stage="load_timeseries_metadata", type="aligned/non_aligned", from="mem/disk" | Timer | 加载 TimeseriesMetadata 耗时 |
+| series_scan_cost | stage="read_timeseries_metadata", type="", from="cache/file" | Timer | 读取一个文件的 Metadata 耗时 |
+| series_scan_cost | stage="timeseries_metadata_modification", type="aligned/non_aligned", from="null" | Timer | 过滤删除的 TimeseriesMetadata 耗时 |
+| series_scan_cost | stage="load_chunk_metadata_list", type="aligned/non_aligned", from="mem/disk" | Timer | 加载 ChunkMetadata 列表耗时 |
+| series_scan_cost | stage="chunk_metadata_modification", type="aligned/non_aligned", from="mem/disk" | Timer | 过滤删除的 ChunkMetadata 耗时 |
+| series_scan_cost | stage="chunk_metadata_filter", type="aligned/non_aligned", from="mem/disk" | Timer | 根据查询过滤条件过滤 ChunkMetadata 耗时 |
+| series_scan_cost | stage="construct_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | 构造 ChunkReader 耗时 |
+| series_scan_cost | stage="read_chunk", type="", from="cache/file" | Timer | 读取 Chunk 的耗时 |
+| series_scan_cost | stage="init_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | 初始化 ChunkReader（构造 PageReader） 耗时 |
+| series_scan_cost | stage="build_tsblock_from_page_reader", type="aligned/non_aligned", from="mem/disk" | Timer | 从 PageReader 构造 Tsblock 耗时 |
+| series_scan_cost | stage="build_tsblock_from_merge_reader", type="aligned/non_aligned", from="null" | Timer | 从 MergeReader 构造 Tsblock （解乱序数据）耗时 |
+
### 4.3. Normal 级别监控指标
#### 4.3.1. 集群
[iotdb] 08/10: fix data_exchange_count tag name
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addMetricDoc
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e92addd65c2e2c5c27aa48c5cbb066fa12a6fc5d
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Jan 12 15:30:46 2023 +0800
fix data_exchange_count tag name
---
.../iotdb/db/mpp/metric/DataExchangeCountMetricSet.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
index 5d8ef839f4..153270cf79 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
@@ -51,7 +51,7 @@ public class DataExchangeCountMetricSet implements IMetricSet {
new MetricInfo(
MetricType.HISTOGRAM,
metric,
- Tag.OPERATION.toString(),
+ Tag.NAME.toString(),
"send_new_data_block_num",
Tag.TYPE.toString(),
"caller"));
@@ -60,7 +60,7 @@ public class DataExchangeCountMetricSet implements IMetricSet {
new MetricInfo(
MetricType.HISTOGRAM,
metric,
- Tag.OPERATION.toString(),
+ Tag.NAME.toString(),
"send_new_data_block_num",
Tag.TYPE.toString(),
"server"));
@@ -69,7 +69,7 @@ public class DataExchangeCountMetricSet implements IMetricSet {
new MetricInfo(
MetricType.HISTOGRAM,
metric,
- Tag.OPERATION.toString(),
+ Tag.NAME.toString(),
"on_acknowledge_data_block_num",
Tag.TYPE.toString(),
"caller"));
@@ -78,7 +78,7 @@ public class DataExchangeCountMetricSet implements IMetricSet {
new MetricInfo(
MetricType.HISTOGRAM,
metric,
- Tag.OPERATION.toString(),
+ Tag.NAME.toString(),
"on_acknowledge_data_block_num",
Tag.TYPE.toString(),
"server"));
@@ -87,7 +87,7 @@ public class DataExchangeCountMetricSet implements IMetricSet {
new MetricInfo(
MetricType.HISTOGRAM,
metric,
- Tag.OPERATION.toString(),
+ Tag.NAME.toString(),
"get_data_block_num",
Tag.TYPE.toString(),
"caller"));
@@ -96,7 +96,7 @@ public class DataExchangeCountMetricSet implements IMetricSet {
new MetricInfo(
MetricType.HISTOGRAM,
metric,
- Tag.OPERATION.toString(),
+ Tag.NAME.toString(),
"get_data_block_num",
Tag.TYPE.toString(),
"server"));