You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bs...@apache.org on 2018/11/26 22:24:27 UTC
[geode] 02/02: GEODE-6035 Increase backlog for peer-to-peer
connection formation
This is an automated email from the ASF dual-hosted git repository.
bschuchardt pushed a commit to branch feature/GEODE-6035
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 6a876c79e8248c4d226f9b6d0b6c76e6cdf783b6
Author: Bruce Schuchardt <bs...@pivotal.io>
AuthorDate: Mon Nov 26 14:12:16 2018 -0800
GEODE-6035 Increase backlog for peer-to-peer connection formation
Adding statistics for Linux to track TCP/IP backlog issues.
tcpExtSynCookiesRecv The number of TCP/IP SYN cookies received due to a full server socket backlog.
If this is non-zero consider disabling SYN cookies because they form sub-optimal connections.
units: cookies received
tcpExtSynCookiesSent The number of TCP/IP SYN cookies sent due to a full server socket backlog.
If this is non-zero consider disabling SYN cookies because they form sub-optimal connections.
units: cookies sent
tcpExtListenDrops The number of TCP/IP connection requests that have been dropped due to a full backlog.
If this is large increase the OS SOMAXCONN setting and increase socket backlog settings.
units: requests
tcpExtListenOverflows The number of TCP/IP connection requests that could not be queued due to a small backlog.
These are either dropped (tcpExtListenDrops) or handled via cookies (tcpSynCookiesSent).
In either case you should consider increasing SOMAXCONN and increasing backlog settings.
units: requests
soMaxConn Maximum TCP/IP server socket connection request backlog
units: connection requests
---
.../statistics/platform/LinuxProcFsStatistics.java | 51 ++++++++++++++++++++++
.../statistics/platform/LinuxSystemStats.java | 31 +++++++++++++
2 files changed, 82 insertions(+)
diff --git a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java
index 395ce0f..56c0068 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxProcFsStatistics.java
@@ -200,6 +200,7 @@ public class LinuxProcFsStatistics {
getMemInfo(ints);
getDiskStats(longs);
getNetStats(longs);
+ getNetStatStats(longs, ints);
if (hasProcVmStat) {
getVmStats(longs);
}
@@ -334,6 +335,56 @@ public class LinuxProcFsStatistics {
}
}
+/*
+ * TcpExt:=0 SyncookiesSent=1
+ * ListenOverflows=20 ListenDrops=21
+ */
+ private static void getNetStatStats(long[] longs, int[] ints) {
+ InputStreamReader isr;
+ BufferedReader br = null;
+ try {
+ isr = new InputStreamReader(new FileInputStream("/proc/net/netstat"));
+ br = new BufferedReader(isr);
+ String line;
+ do {
+ br.readLine(); // header
+ line = br.readLine();
+ } while (line != null && !line.startsWith("TcpExt:"));
+
+ st.setString(line);
+ st.skipTokens(1);
+ long tcpSyncookiesSent = st.nextTokenAsLong();
+ long tcpSyncookiesRecv = st.nextTokenAsLong();
+ st.skipTokens(17);
+ long tcpListenOverflows = st.nextTokenAsLong();
+ long tcpListenDrops = st.nextTokenAsLong();
+
+ longs[LinuxSystemStats.tcpExtSynCookiesRecvLONG] = tcpSyncookiesRecv;
+ longs[LinuxSystemStats.tcpExtSynCookiesSentLONG] = tcpSyncookiesSent;
+ longs[LinuxSystemStats.tcpExtListenDropsLONG] = tcpListenDrops;
+ longs[LinuxSystemStats.tcpExtListenOverflowsLONG] = tcpListenOverflows;
+
+ br.close();
+ isr = new InputStreamReader(new FileInputStream("/proc/sys/net/core/somaxconn"));
+ br = new BufferedReader(isr);
+ line = br.readLine();
+ st.setString(line);
+ int soMaxConn = st.nextTokenAsInt();
+
+ ints[LinuxSystemStats.tcpSOMaxConnINT] = soMaxConn;
+
+ } catch (NoSuchElementException nsee) {
+ } catch (IOException ioe) {
+ } finally {
+ st.releaseResources();
+ if (br != null)
+ try {
+ br.close();
+ } catch (IOException ignore) {
+ }
+ }
+ }
+
/*
* Inter-| Receive | Transmit face |bytes packets errs drop fifo frame compressed multicast|bytes
* packets errs drop fifo colls carrier compressed lo:1908275823 326949246 0 0 0 0 0 0 1908275823
diff --git a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java
index 662c95f..9da3715 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/statistics/platform/LinuxSystemStats.java
@@ -49,6 +49,7 @@ public class LinuxSystemStats {
static final int dirtyMemoryINT = 17;
static final int cpuNonUserINT = 18;
static final int cpuStealINT = 19;
+ static final int tcpSOMaxConnINT = 20;
static final int loopbackPacketsLONG = 0;
static final int loopbackBytesLONG = 1;
@@ -78,6 +79,11 @@ public class LinuxSystemStats {
static final int iosInProgressLONG = 25;
static final int timeIosInProgressLONG = 26;
static final int ioTimeLONG = 27;
+ static final int tcpExtSynCookiesRecvLONG = 28;
+ static final int tcpExtSynCookiesSentLONG = 29;
+ static final int tcpExtListenDropsLONG = 30;
+ static final int tcpExtListenOverflowsLONG = 31;
+
static final int loadAverage1DOUBLE = 0;
static final int loadAverage15DOUBLE = 1;
@@ -149,6 +155,9 @@ public class LinuxSystemStats {
f.createIntGauge("cpuSteal",
"Steal time is the amount of time the operating system wanted to execute, but was not allowed to by the hypervisor.",
"%"),
+ f.createIntGauge("soMaxConn",
+ "Maximum TCP/IP server socket connection request backlog",
+ "connection requests"),
f.createLongCounter("loopbackPackets",
"The number of network packets sent (or received) on the loopback interface",
@@ -219,6 +228,23 @@ public class LinuxSystemStats {
f.createLongCounter("diskTime",
"The total number of milliseconds that measures both completed disk operations and any accumulating backlog of in progress ops.",
"milliseconds"),
+ f.createLongCounter("tcpExtSynCookiesRecv",
+ "The number of TCP/IP SYN cookies received due to a full server socket backlog. "
+ + "If this is non-zero consider disabling SYN cookies because they form sub-optimal connections.",
+ "cookies received"),
+ f.createLongCounter("tcpExtSynCookiesSent",
+ "The number of TCP/IP SYN cookies sent due to a full server socket backlog. "
+ + "If this is non-zero consider disabling SYN cookies because they form sub-optimal connections.",
+ "cookies sent"),
+ f.createLongCounter("tcpExtListenDrops",
+ "The number of TCP/IP connection requests that have been dropped due to a full backlog. "
+ + "If this is large increase the OS SOMAXCONN setting and increase socket backlog settings",
+ "requests"),
+ f.createLongCounter("tcpExtListenOverflows",
+ "The number of TCP/IP connection requests that could not be queued due to a small backlog. "
+ + "These are either dropped (tcpExtListenDrops) or handled via cookies (tcpSynCookiesSent). "
+ + "In either case you should consider increasing SOMAXCONN and increasing backlog settings.",
+ "requests"),
f.createDoubleGauge("loadAverage1",
@@ -251,6 +277,7 @@ public class LinuxSystemStats {
checkOffset("dirtyMemory", dirtyMemoryINT);
checkOffset("cpuNonUser", cpuNonUserINT);
checkOffset("cpuSteal", cpuStealINT);
+ checkOffset("soMaxConn", tcpSOMaxConnINT);
checkOffset("loopbackPackets", loopbackPacketsLONG);
checkOffset("loopbackBytes", loopbackBytesLONG);
@@ -280,6 +307,10 @@ public class LinuxSystemStats {
checkOffset("diskOpsInProgress", iosInProgressLONG);
checkOffset("diskTimeInProgress", timeIosInProgressLONG);
checkOffset("diskTime", ioTimeLONG);
+ checkOffset("tcpExtSynCookiesRecv", tcpExtSynCookiesRecvLONG);
+ checkOffset("tcpExtSynCookiesSent", tcpExtSynCookiesSentLONG);
+ checkOffset("tcpExtListenDrops", tcpExtListenDropsLONG);
+ checkOffset("tcpExtListenOverflows", tcpExtListenOverflowsLONG);
checkOffset("loadAverage1", loadAverage1DOUBLE);
checkOffset("loadAverage15", loadAverage15DOUBLE);