You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ma...@apache.org on 2022/12/19 09:59:39 UTC
[pulsar] branch introduce_oshi created (now c9cd5322994)
This is an automated email from the ASF dual-hosted git repository.
mattisonchao pushed a change to branch introduce_oshi
in repository https://gitbox.apache.org/repos/asf/pulsar.git
at c9cd5322994 Introduce oshi
This branch includes the following new commits:
new c9cd5322994 Introduce oshi
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[pulsar] 01/01: Introduce oshi
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mattisonchao pushed a commit to branch introduce_oshi
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit c9cd5322994460a7bdd9a03a7bdcac00c5164afc
Author: mattisonchao <ma...@apache.org>
AuthorDate: Mon Dec 19 17:58:46 2022 +0800
Introduce oshi
---
pulsar-broker/pom.xml | 5 +++++
.../pulsar/broker/loadbalance/LinuxInfoUtils.java | 17 +++++++++++---
.../loadbalance/impl/LinuxBrokerHostUsageImpl.java | 26 +++++++++++++++-------
3 files changed, 37 insertions(+), 11 deletions(-)
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 136bd670d5e..2a213ec2905 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -426,6 +426,11 @@
<artifactId>pulsar-package-filesystem-storage</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.github.oshi</groupId>
+ <artifactId>oshi-core-java11</artifactId>
+ <version>6.4.0</version>
+ </dependency>
</dependencies>
<build>
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
index 42ef264b6db..a7a759a70f6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java
@@ -37,6 +37,8 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.pulsar.broker.BitRateUnit;
+import oshi.SystemInfo;
+import oshi.hardware.NetworkIF;
@Slf4j
public class LinuxInfoUtils {
@@ -55,6 +57,7 @@ public class LinuxInfoUtils {
/**
* Determine whether the OS is the linux kernel.
+ *
* @return Whether the OS is the linux kernel
*/
public static boolean isLinux() {
@@ -75,6 +78,7 @@ public class LinuxInfoUtils {
/**
* Get total cpu limit.
+ *
* @param isCGroupsEnabled Whether CGroup is enabled
* @return Total cpu limit
*/
@@ -97,6 +101,7 @@ public class LinuxInfoUtils {
/**
* Get CGroup cpu usage.
+ *
* @return Cpu usage
*/
public static double getCpuUsageForCGroup() {
@@ -145,6 +150,7 @@ public class LinuxInfoUtils {
/**
* Determine whether the VM has physical nic.
+ *
* @param nicPath Nic path
* @return whether The VM has physical nic.
*/
@@ -166,6 +172,7 @@ public class LinuxInfoUtils {
/**
* Determine whether nic is usable.
+ *
* @param nicPath Nic path
* @return whether nic is usable.
*/
@@ -190,7 +197,8 @@ public class LinuxInfoUtils {
/**
* Get all physical nic limit.
- * @param nics All nic path
+ *
+ * @param nics All nic path
* @param bitRateUnit Bit rate unit
* @return Total nic limit
*/
@@ -207,8 +215,9 @@ public class LinuxInfoUtils {
/**
* Get all physical nic usage.
- * @param nics All nic path
- * @param type Nic's usage type: transport, receive
+ *
+ * @param nics All nic path
+ * @param type Nic's usage type: transport, receive
* @param bitRateUnit Bit rate unit
* @return Total nic usage
*/
@@ -225,6 +234,7 @@ public class LinuxInfoUtils {
/**
* Get paths of all usable physical nic.
+ *
* @return All usable physical nic paths.
*/
public static List<String> getUsablePhysicalNICs() {
@@ -241,6 +251,7 @@ public class LinuxInfoUtils {
/**
* Check this VM has nic speed.
+ *
* @return Whether the VM has nic speed
*/
public static boolean checkHasNicSpeeds() {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
index 318f37f7f7a..1162b6a759e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.BitRateUnit;
import org.apache.pulsar.broker.PulsarService;
@@ -41,6 +42,9 @@ import org.apache.pulsar.broker.loadbalance.BrokerHostUsage;
import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils;
import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage;
import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage;
+import oshi.SystemInfo;
+import oshi.hardware.HardwareAbstractionLayer;
+import oshi.hardware.NetworkIF;
/**
@@ -60,9 +64,9 @@ public class LinuxBrokerHostUsageImpl implements BrokerHostUsage {
public LinuxBrokerHostUsageImpl(PulsarService pulsar) {
this(
- pulsar.getConfiguration().getLoadBalancerHostUsageCheckIntervalMinutes(),
- pulsar.getConfiguration().getLoadBalancerOverrideBrokerNicSpeedGbps(),
- pulsar.getLoadManagerExecutor()
+ pulsar.getConfiguration().getLoadBalancerHostUsageCheckIntervalMinutes(),
+ pulsar.getConfiguration().getLoadBalancerOverrideBrokerNicSpeedGbps(),
+ pulsar.getLoadManagerExecutor()
);
}
@@ -88,10 +92,16 @@ public class LinuxBrokerHostUsageImpl implements BrokerHostUsage {
@Override
public void calculateBrokerHostUsage() {
- List<String> nics = getUsablePhysicalNICs();
- double totalNicLimit = getTotalNicLimitWithConfiguration(nics);
- double totalNicUsageTx = getTotalNicUsage(nics, NICUsageType.TX, BitRateUnit.Kilobit);
- double totalNicUsageRx = getTotalNicUsage(nics, NICUsageType.RX, BitRateUnit.Kilobit);
+ HardwareAbstractionLayer hardware = new SystemInfo().getHardware();
+ List<NetworkIF> physicalNics = hardware.getNetworkIFs().stream()
+ .filter(ifs -> !ifs.queryNetworkInterface().isVirtual())
+ .filter(ifs -> ifs.getIfOperStatus() == NetworkIF.IfOperStatus.UP ||
+ ifs.getIfOperStatus() == NetworkIF.IfOperStatus.UNKNOWN ||
+ ifs.getIfOperStatus() == NetworkIF.IfOperStatus.DORMANT)
+ .collect(Collectors.toList());
+ long totalNicLimit = physicalNics.stream().mapToLong(NetworkIF::getSpeed).sum();
+ long totalNicUsageTx = physicalNics.stream().mapToLong(NetworkIF::getBytesSent).sum();
+ long totalNicUsageRx = physicalNics.stream().mapToLong(NetworkIF::getBytesRecv).sum();
double totalCpuLimit = getTotalCpuLimit(isCGroupsEnabled);
long now = System.currentTimeMillis();
double elapsedSeconds = (now - lastCollection) / 1000d;
@@ -153,7 +163,7 @@ public class LinuxBrokerHostUsageImpl implements BrokerHostUsage {
* cpu user nice system idle iowait irq softirq steal guest guest_nice
* cpu 317808 128 58637 2503692 7634 0 13472 0 0 0
* </pre>
- *
+ * <p>
* Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this
* far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal.
*/