You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2023/03/30 07:59:27 UTC
[iotdb] branch cherry-pick-net-dashboard created (now bfdb906ba5)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a change to branch cherry-pick-net-dashboard
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at bfdb906ba5 cherry pick network metrics
This branch includes the following new commits:
new bfdb906ba5 cherry pick network metrics
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: cherry pick network metrics
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch cherry-pick-net-dashboard
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bfdb906ba54a6d7cf1674e24a3e534cfc709449f
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Sun Mar 12 22:49:55 2023 +0800
cherry pick network metrics
---
.../iotdb/confignode/service/ConfigNode.java | 4 +
.../metrics/metricsets/net/INetMetricManager.java | 58 ++++++
.../metricsets/net/LinuxNetMetricManager.java | 197 +++++++++++++++++++++
.../metricsets/net/MacNetMetricManager.java | 22 +++
.../iotdb/metrics/metricsets/net/NetMetrics.java | 98 ++++++++++
.../metricsets/net/WindowsNetMetricManager.java | 22 +++
.../db/service/metrics/DataNodeMetricsHelper.java | 2 +
7 files changed, 403 insertions(+)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
index 090d1f4bb0..f69e2c13eb 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.metricsets.disk.DiskMetrics;
import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics;
import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics;
+import org.apache.iotdb.metrics.metricsets.net.NetMetrics;
import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
@@ -229,6 +230,9 @@ public class ConfigNode implements ConfigNodeMBean {
MetricService.getInstance().addMetricSet(new ProcessMetrics());
MetricService.getInstance().addMetricSet(new SystemMetrics(false));
MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.CN_ROLE));
+ MetricService.getInstance().addMetricSet(new NetMetrics());
+
+ LOGGER.info("Successfully setup internal services.");
}
private void initConfigManager() {
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java
new file mode 100644
index 0000000000..f6ae15d37c
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+public interface INetMetricManager {
+ static INetMetricManager getNetMetricManager() {
+ String os = System.getProperty("os.name").toLowerCase();
+
+ if (os.startsWith("windows")) {
+ return new WindowsNetMetricManager();
+ } else if (os.startsWith("linux")) {
+ return new LinuxNetMetricManager();
+ } else {
+ return new MacNetMetricManager();
+ }
+ }
+
+ default Map<String, Long> getReceivedByte() {
+ return Collections.emptyMap();
+ }
+
+ default Map<String, Long> getTransmittedBytes() {
+ return Collections.emptyMap();
+ }
+
+ default Map<String, Long> getReceivedPackets() {
+ return Collections.emptyMap();
+ }
+
+ default Map<String, Long> getTransmittedPackets() {
+ return Collections.emptyMap();
+ }
+
+ default Set<String> getIfaceSet() {
+ return Collections.emptySet();
+ }
+}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java
new file mode 100644
index 0000000000..4e3fab7061
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class LinuxNetMetricManager implements INetMetricManager {
+ private final Logger log = LoggerFactory.getLogger(LinuxNetMetricManager.class);
+
+ @SuppressWarnings("squid:S1075")
+ private static final String IFACE_ID_PATH = "/sys/class/net/";
+
+ @SuppressWarnings("squid:S1075")
+ private static final String NET_STATUS_PATH = "/proc/net/dev";
+
+ private static final String BYTES = "bytes";
+ private static final String PACKETS = "packets";
+ private static final long UPDATE_INTERVAL = 10_000L;
+
+ private static final int IFACE_NAME_INDEX = 0;
+ // initialized after reading status file
+ private int receivedBytesIndex = 0;
+ private int transmittedBytesIndex = 0;
+ private int receivedPacketsIndex = 0;
+ private int transmittedPacketsIndex = 0;
+ private Set<String> ifaceSet;
+
+ private final Map<String, Long> receivedBytesMapForIface;
+ private final Map<String, Long> transmittedBytesMapForIface;
+ private final Map<String, Long> receivedPacketsMapForIface;
+ private final Map<String, Long> transmittedPacketsMapForIface;
+
+ public LinuxNetMetricManager() {
+ collectIfaces();
+ // leave one entry to avoid hashmap resizing
+ receivedBytesMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ transmittedBytesMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ receivedPacketsMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ transmittedPacketsMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ collectNetStatusIndex();
+ }
+
+ private long lastUpdateTime = 0L;
+
+ @Override
+ public Set<String> getIfaceSet() {
+ checkUpdate();
+ return ifaceSet;
+ }
+
+ @Override
+ public Map<String, Long> getReceivedByte() {
+ checkUpdate();
+ return receivedBytesMapForIface;
+ }
+
+ @Override
+ public Map<String, Long> getTransmittedBytes() {
+ checkUpdate();
+ return transmittedBytesMapForIface;
+ }
+
+ @Override
+ public Map<String, Long> getReceivedPackets() {
+ checkUpdate();
+ return receivedPacketsMapForIface;
+ }
+
+ @Override
+ public Map<String, Long> getTransmittedPackets() {
+ checkUpdate();
+ return transmittedPacketsMapForIface;
+ }
+
+ private void checkUpdate() {
+ if (System.currentTimeMillis() - lastUpdateTime >= UPDATE_INTERVAL) {
+ updateNetStatus();
+ }
+ }
+
+ private void collectIfaces() {
+ File ifaceIdFolder = new File(IFACE_ID_PATH);
+ if (!ifaceIdFolder.exists()) {
+ ifaceSet = Collections.emptySet();
+ log.warn("Cannot find {}", IFACE_ID_PATH);
+ return;
+ }
+ ifaceSet =
+ new ArrayList<>(Arrays.asList(Objects.requireNonNull(ifaceIdFolder.listFiles())))
+ .stream().map(File::getName).collect(Collectors.toSet());
+ }
+
+ private void collectNetStatusIndex() {
+ File netStatusFile = new File(NET_STATUS_PATH);
+ if (!netStatusFile.exists()) {
+ log.warn("Cannot find {}", NET_STATUS_PATH);
+ return;
+ }
+ try (FileInputStream inputStream = new FileInputStream(netStatusFile)) {
+ Scanner scanner = new Scanner(inputStream);
+ // skip the first line
+ scanner.nextLine();
+ String headerLine = scanner.nextLine();
+ String[] seperatedHeaderLine = headerLine.split("\\|");
+ String[] receiveStatusHeader = seperatedHeaderLine[1].split("\\s+");
+ String[] transmitStatusHeader = seperatedHeaderLine[2].split("\\s+");
+ for (int i = 0, length = receiveStatusHeader.length; i < length; ++i) {
+ if (receiveStatusHeader[i].equals(BYTES)) {
+ receivedBytesIndex = i + 1;
+ } else if (receiveStatusHeader[i].equals(PACKETS)) {
+ receivedPacketsIndex = i + 1;
+ }
+ }
+ for (int i = 0, length = transmitStatusHeader.length; i < length; ++i) {
+ if (transmitStatusHeader[i].equals(BYTES)) {
+ transmittedBytesIndex = i + length + 1;
+ } else if (transmitStatusHeader[i].equals(PACKETS)) {
+ transmittedPacketsIndex = i + length + 1;
+ }
+ }
+ } catch (IOException e) {
+ log.error("Meets exception when reading {}", NET_STATUS_PATH, e);
+ }
+ }
+
+ private void updateNetStatus() {
+ lastUpdateTime = System.currentTimeMillis();
+ File netStatusFile = new File(NET_STATUS_PATH);
+ if (!netStatusFile.exists()) {
+ return;
+ }
+ try (FileInputStream inputStream = new FileInputStream(netStatusFile)) {
+ Scanner scanner = new Scanner(inputStream);
+ // skip the starting two lines
+ // because they are the meta info
+ scanner.nextLine();
+ scanner.nextLine();
+
+ // reading the actual status info for iface
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ // we wrap the string array as array list to execute the removing step
+ List<String> statusInfoAsList = new ArrayList<>(Arrays.asList(line.split("\\s")));
+ // remove all empty string
+ statusInfoAsList.removeIf(x -> x.equals(""));
+
+ String iface = statusInfoAsList.get(IFACE_NAME_INDEX);
+ // since the read iface format is "IFACE:"
+ // we need to remove the last letter
+ iface = iface.substring(0, iface.length() - 1);
+
+ long receivedBytes = Long.parseLong(statusInfoAsList.get(receivedBytesIndex));
+ receivedBytesMapForIface.put(iface, receivedBytes);
+ long transmittedBytes = Long.parseLong(statusInfoAsList.get(transmittedBytesIndex));
+ transmittedBytesMapForIface.put(iface, transmittedBytes);
+ long receivedPackets = Long.parseLong(statusInfoAsList.get(receivedPacketsIndex));
+ receivedPacketsMapForIface.put(iface, receivedPackets);
+ long transmittedPackets = Long.parseLong(statusInfoAsList.get(transmittedPacketsIndex));
+ transmittedPacketsMapForIface.put(iface, transmittedPackets);
+ }
+ } catch (IOException e) {
+ log.error("Meets error when reading {} for net status", NET_STATUS_PATH, e);
+ }
+ }
+}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java
new file mode 100644
index 0000000000..bb3a31fba2
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+public class MacNetMetricManager implements INetMetricManager {}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java
new file mode 100644
index 0000000000..e1e565c049
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.util.Set;
+
+public class NetMetrics implements IMetricSet {
+ private final INetMetricManager netMetricManager = INetMetricManager.getNetMetricManager();
+
+ private static final String RECEIVED_BYTES = "received_bytes";
+ private static final String RECEIVED_PACKETS = "received_packets";
+ private static final String TRANSMITTED_BYTES = "transmitted_bytes";
+ private static final String TRANSMITTED_PACKETS = "transmitted_packets";
+
+ private static final String TYPE = "type";
+ private static final String IFACE_NAME = "iface_name";
+ private static final String RECEIVE = "receive";
+ private static final String TRANSMIT = "transmit";
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ // metrics for net
+ Set<String> ifaceSet = netMetricManager.getIfaceSet();
+ for (String iface : ifaceSet) {
+ metricService.createAutoGauge(
+ RECEIVED_BYTES,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getReceivedByte().get(iface),
+ TYPE,
+ RECEIVE,
+ IFACE_NAME,
+ iface);
+ metricService.createAutoGauge(
+ TRANSMITTED_BYTES,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getTransmittedBytes().get(iface),
+ TYPE,
+ TRANSMIT,
+ IFACE_NAME,
+ iface);
+ metricService.createAutoGauge(
+ RECEIVED_PACKETS,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getReceivedPackets().get(iface),
+ TYPE,
+ RECEIVE,
+ IFACE_NAME,
+ iface);
+ metricService.createAutoGauge(
+ TRANSMITTED_PACKETS,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getTransmittedPackets().get(iface),
+ TYPE,
+ TRANSMIT,
+ IFACE_NAME,
+ iface);
+ }
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ Set<String> ifaceSet = netMetricManager.getIfaceSet();
+ for (String iface : ifaceSet) {
+ metricService.remove(MetricType.AUTO_GAUGE, RECEIVED_BYTES, TYPE, RECEIVE, IFACE_NAME, iface);
+ metricService.remove(MetricType.AUTO_GAUGE, TRANSMIT, TYPE, TRANSMIT, IFACE_NAME, iface);
+ metricService.remove(
+ MetricType.AUTO_GAUGE, RECEIVED_PACKETS, TYPE, RECEIVE, IFACE_NAME, iface);
+ metricService.remove(
+ MetricType.AUTO_GAUGE, TRANSMITTED_PACKETS, TYPE, TRANSMIT, IFACE_NAME, iface);
+ }
+ }
+}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java
new file mode 100644
index 0000000000..c3ecb4b8d5
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+public class WindowsNetMetricManager implements INetMetricManager {}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index 664b0c3a67..110d4cfa4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.metrics.metricsets.disk.DiskMetrics;
import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics;
import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics;
+import org.apache.iotdb.metrics.metricsets.net.NetMetrics;
public class DataNodeMetricsHelper {
/** Bind predefined metric sets into DataNode. */
@@ -43,6 +44,7 @@ public class DataNodeMetricsHelper {
MetricService.getInstance().addMetricSet(new ProcessMetrics());
MetricService.getInstance().addMetricSet(new SystemMetrics(true));
MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE));
+ MetricService.getInstance().addMetricSet(new NetMetrics());
// bind query related metrics
MetricService.getInstance().addMetricSet(new QueryPlanCostMetricSet());