You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2023/03/30 07:59:28 UTC
[iotdb] 01/01: cherry pick network metrics
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch cherry-pick-net-dashboard
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bfdb906ba54a6d7cf1674e24a3e534cfc709449f
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Sun Mar 12 22:49:55 2023 +0800
cherry pick network metrics
---
.../iotdb/confignode/service/ConfigNode.java | 4 +
.../metrics/metricsets/net/INetMetricManager.java | 58 ++++++
.../metricsets/net/LinuxNetMetricManager.java | 197 +++++++++++++++++++++
.../metricsets/net/MacNetMetricManager.java | 22 +++
.../iotdb/metrics/metricsets/net/NetMetrics.java | 98 ++++++++++
.../metricsets/net/WindowsNetMetricManager.java | 22 +++
.../db/service/metrics/DataNodeMetricsHelper.java | 2 +
7 files changed, 403 insertions(+)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
index 090d1f4bb0..f69e2c13eb 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.metricsets.disk.DiskMetrics;
import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics;
import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics;
+import org.apache.iotdb.metrics.metricsets.net.NetMetrics;
import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
@@ -229,6 +230,9 @@ public class ConfigNode implements ConfigNodeMBean {
MetricService.getInstance().addMetricSet(new ProcessMetrics());
MetricService.getInstance().addMetricSet(new SystemMetrics(false));
MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.CN_ROLE));
+ MetricService.getInstance().addMetricSet(new NetMetrics());
+
+ LOGGER.info("Successfully setup internal services.");
}
private void initConfigManager() {
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java
new file mode 100644
index 0000000000..f6ae15d37c
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/INetMetricManager.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+public interface INetMetricManager {
+ static INetMetricManager getNetMetricManager() {
+ String os = System.getProperty("os.name").toLowerCase();
+
+ if (os.startsWith("windows")) {
+ return new WindowsNetMetricManager();
+ } else if (os.startsWith("linux")) {
+ return new LinuxNetMetricManager();
+ } else {
+ return new MacNetMetricManager();
+ }
+ }
+
+ default Map<String, Long> getReceivedByte() {
+ return Collections.emptyMap();
+ }
+
+ default Map<String, Long> getTransmittedBytes() {
+ return Collections.emptyMap();
+ }
+
+ default Map<String, Long> getReceivedPackets() {
+ return Collections.emptyMap();
+ }
+
+ default Map<String, Long> getTransmittedPackets() {
+ return Collections.emptyMap();
+ }
+
+ default Set<String> getIfaceSet() {
+ return Collections.emptySet();
+ }
+}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java
new file mode 100644
index 0000000000..4e3fab7061
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/LinuxNetMetricManager.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class LinuxNetMetricManager implements INetMetricManager {
+ private final Logger log = LoggerFactory.getLogger(LinuxNetMetricManager.class);
+
+ @SuppressWarnings("squid:S1075")
+ private static final String IFACE_ID_PATH = "/sys/class/net/";
+
+ @SuppressWarnings("squid:S1075")
+ private static final String NET_STATUS_PATH = "/proc/net/dev";
+
+ private static final String BYTES = "bytes";
+ private static final String PACKETS = "packets";
+ private static final long UPDATE_INTERVAL = 10_000L;
+
+ private static final int IFACE_NAME_INDEX = 0;
+ // initialized after reading status file
+ private int receivedBytesIndex = 0;
+ private int transmittedBytesIndex = 0;
+ private int receivedPacketsIndex = 0;
+ private int transmittedPacketsIndex = 0;
+ private Set<String> ifaceSet;
+
+ private final Map<String, Long> receivedBytesMapForIface;
+ private final Map<String, Long> transmittedBytesMapForIface;
+ private final Map<String, Long> receivedPacketsMapForIface;
+ private final Map<String, Long> transmittedPacketsMapForIface;
+
+ public LinuxNetMetricManager() {
+ collectIfaces();
+ // leave one entry to avoid hashmap resizing
+ receivedBytesMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ transmittedBytesMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ receivedPacketsMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ transmittedPacketsMapForIface = new HashMap<>(ifaceSet.size() + 1, 1);
+ collectNetStatusIndex();
+ }
+
+ private long lastUpdateTime = 0L;
+
+ @Override
+ public Set<String> getIfaceSet() {
+ checkUpdate();
+ return ifaceSet;
+ }
+
+ @Override
+ public Map<String, Long> getReceivedByte() {
+ checkUpdate();
+ return receivedBytesMapForIface;
+ }
+
+ @Override
+ public Map<String, Long> getTransmittedBytes() {
+ checkUpdate();
+ return transmittedBytesMapForIface;
+ }
+
+ @Override
+ public Map<String, Long> getReceivedPackets() {
+ checkUpdate();
+ return receivedPacketsMapForIface;
+ }
+
+ @Override
+ public Map<String, Long> getTransmittedPackets() {
+ checkUpdate();
+ return transmittedPacketsMapForIface;
+ }
+
+ private void checkUpdate() {
+ if (System.currentTimeMillis() - lastUpdateTime >= UPDATE_INTERVAL) {
+ updateNetStatus();
+ }
+ }
+
+ private void collectIfaces() {
+ File ifaceIdFolder = new File(IFACE_ID_PATH);
+ if (!ifaceIdFolder.exists()) {
+ ifaceSet = Collections.emptySet();
+ log.warn("Cannot find {}", IFACE_ID_PATH);
+ return;
+ }
+ ifaceSet =
+ new ArrayList<>(Arrays.asList(Objects.requireNonNull(ifaceIdFolder.listFiles())))
+ .stream().map(File::getName).collect(Collectors.toSet());
+ }
+
+ private void collectNetStatusIndex() {
+ File netStatusFile = new File(NET_STATUS_PATH);
+ if (!netStatusFile.exists()) {
+ log.warn("Cannot find {}", NET_STATUS_PATH);
+ return;
+ }
+ try (FileInputStream inputStream = new FileInputStream(netStatusFile)) {
+ Scanner scanner = new Scanner(inputStream);
+ // skip the first line
+ scanner.nextLine();
+ String headerLine = scanner.nextLine();
+ String[] seperatedHeaderLine = headerLine.split("\\|");
+ String[] receiveStatusHeader = seperatedHeaderLine[1].split("\\s+");
+ String[] transmitStatusHeader = seperatedHeaderLine[2].split("\\s+");
+ for (int i = 0, length = receiveStatusHeader.length; i < length; ++i) {
+ if (receiveStatusHeader[i].equals(BYTES)) {
+ receivedBytesIndex = i + 1;
+ } else if (receiveStatusHeader[i].equals(PACKETS)) {
+ receivedPacketsIndex = i + 1;
+ }
+ }
+ for (int i = 0, length = transmitStatusHeader.length; i < length; ++i) {
+ if (transmitStatusHeader[i].equals(BYTES)) {
+ transmittedBytesIndex = i + length + 1;
+ } else if (transmitStatusHeader[i].equals(PACKETS)) {
+ transmittedPacketsIndex = i + length + 1;
+ }
+ }
+ } catch (IOException e) {
+ log.error("Meets exception when reading {}", NET_STATUS_PATH, e);
+ }
+ }
+
+ private void updateNetStatus() {
+ lastUpdateTime = System.currentTimeMillis();
+ File netStatusFile = new File(NET_STATUS_PATH);
+ if (!netStatusFile.exists()) {
+ return;
+ }
+ try (FileInputStream inputStream = new FileInputStream(netStatusFile)) {
+ Scanner scanner = new Scanner(inputStream);
+ // skip the starting two lines
+ // because they are the meta info
+ scanner.nextLine();
+ scanner.nextLine();
+
+ // reading the actual status info for iface
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ // we wrap the string array as array list to execute the removing step
+ List<String> statusInfoAsList = new ArrayList<>(Arrays.asList(line.split("\\s")));
+ // remove all empty string
+ statusInfoAsList.removeIf(x -> x.equals(""));
+
+ String iface = statusInfoAsList.get(IFACE_NAME_INDEX);
+ // since the read iface format is "IFACE:"
+ // we need to remove the last letter
+ iface = iface.substring(0, iface.length() - 1);
+
+ long receivedBytes = Long.parseLong(statusInfoAsList.get(receivedBytesIndex));
+ receivedBytesMapForIface.put(iface, receivedBytes);
+ long transmittedBytes = Long.parseLong(statusInfoAsList.get(transmittedBytesIndex));
+ transmittedBytesMapForIface.put(iface, transmittedBytes);
+ long receivedPackets = Long.parseLong(statusInfoAsList.get(receivedPacketsIndex));
+ receivedPacketsMapForIface.put(iface, receivedPackets);
+ long transmittedPackets = Long.parseLong(statusInfoAsList.get(transmittedPacketsIndex));
+ transmittedPacketsMapForIface.put(iface, transmittedPackets);
+ }
+ } catch (IOException e) {
+ log.error("Meets error when reading {} for net status", NET_STATUS_PATH, e);
+ }
+ }
+}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java
new file mode 100644
index 0000000000..bb3a31fba2
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/MacNetMetricManager.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+public class MacNetMetricManager implements INetMetricManager {}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java
new file mode 100644
index 0000000000..e1e565c049
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/NetMetrics.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.util.Set;
+
+public class NetMetrics implements IMetricSet {
+ private final INetMetricManager netMetricManager = INetMetricManager.getNetMetricManager();
+
+ private static final String RECEIVED_BYTES = "received_bytes";
+ private static final String RECEIVED_PACKETS = "received_packets";
+ private static final String TRANSMITTED_BYTES = "transmitted_bytes";
+ private static final String TRANSMITTED_PACKETS = "transmitted_packets";
+
+ private static final String TYPE = "type";
+ private static final String IFACE_NAME = "iface_name";
+ private static final String RECEIVE = "receive";
+ private static final String TRANSMIT = "transmit";
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ // metrics for net
+ Set<String> ifaceSet = netMetricManager.getIfaceSet();
+ for (String iface : ifaceSet) {
+ metricService.createAutoGauge(
+ RECEIVED_BYTES,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getReceivedByte().get(iface),
+ TYPE,
+ RECEIVE,
+ IFACE_NAME,
+ iface);
+ metricService.createAutoGauge(
+ TRANSMITTED_BYTES,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getTransmittedBytes().get(iface),
+ TYPE,
+ TRANSMIT,
+ IFACE_NAME,
+ iface);
+ metricService.createAutoGauge(
+ RECEIVED_PACKETS,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getReceivedPackets().get(iface),
+ TYPE,
+ RECEIVE,
+ IFACE_NAME,
+ iface);
+ metricService.createAutoGauge(
+ TRANSMITTED_PACKETS,
+ MetricLevel.IMPORTANT,
+ netMetricManager,
+ x -> x.getTransmittedPackets().get(iface),
+ TYPE,
+ TRANSMIT,
+ IFACE_NAME,
+ iface);
+ }
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ Set<String> ifaceSet = netMetricManager.getIfaceSet();
+ for (String iface : ifaceSet) {
+ metricService.remove(MetricType.AUTO_GAUGE, RECEIVED_BYTES, TYPE, RECEIVE, IFACE_NAME, iface);
+ metricService.remove(MetricType.AUTO_GAUGE, TRANSMIT, TYPE, TRANSMIT, IFACE_NAME, iface);
+ metricService.remove(
+ MetricType.AUTO_GAUGE, RECEIVED_PACKETS, TYPE, RECEIVE, IFACE_NAME, iface);
+ metricService.remove(
+ MetricType.AUTO_GAUGE, TRANSMITTED_PACKETS, TYPE, TRANSMIT, IFACE_NAME, iface);
+ }
+ }
+}
diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java
new file mode 100644
index 0000000000..c3ecb4b8d5
--- /dev/null
+++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/WindowsNetMetricManager.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.metrics.metricsets.net;
+
+public class WindowsNetMetricManager implements INetMetricManager {}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index 664b0c3a67..110d4cfa4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.metrics.metricsets.disk.DiskMetrics;
import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics;
import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics;
+import org.apache.iotdb.metrics.metricsets.net.NetMetrics;
public class DataNodeMetricsHelper {
/** Bind predefined metric sets into DataNode. */
@@ -43,6 +44,7 @@ public class DataNodeMetricsHelper {
MetricService.getInstance().addMetricSet(new ProcessMetrics());
MetricService.getInstance().addMetricSet(new SystemMetrics(true));
MetricService.getInstance().addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE));
+ MetricService.getInstance().addMetricSet(new NetMetrics());
// bind query related metrics
MetricService.getInstance().addMetricSet(new QueryPlanCostMetricSet());