You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/10 17:42:21 UTC
[16/21] hadoop git commit: HADOOP-12210. Collect network usage on the
node. Contributed by Robert Grandl
HADOOP-12210. Collect network usage on the node. Contributed by Robert Grandl
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a0752d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a0752d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a0752d8
Branch: refs/heads/HADOOP-12111
Commit: 1a0752d85a15499d120b4a79af9bd740fcd1f8e0
Parents: 0e602fa
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Jul 6 17:28:20 2015 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Jul 9 17:48:43 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../java/org/apache/hadoop/util/SysInfo.java | 12 +++
.../org/apache/hadoop/util/SysInfoLinux.java | 93 +++++++++++++++++++-
.../org/apache/hadoop/util/SysInfoWindows.java | 15 ++++
.../apache/hadoop/util/TestSysInfoLinux.java | 40 ++++++++-
.../gridmix/DummyResourceCalculatorPlugin.java | 19 ++++
.../yarn/util/ResourceCalculatorPlugin.java | 16 ++++
7 files changed, 195 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d9a9eba..3d4f1e4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -693,6 +693,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12180. Move ResourceCalculatorPlugin from YARN to Common.
(Chris Douglas via kasha)
+ HADOOP-12210. Collect network usage on the node (Robert Grandl via cdouglas)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
index ec7fb24..24b339d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
@@ -108,4 +108,16 @@ public abstract class SysInfo {
*/
public abstract float getCpuUsage();
+ /**
+ * Obtain the aggregated number of bytes read over the network.
+ * @return total number of bytes read.
+ */
+ public abstract long getNetworkBytesRead();
+
+ /**
+ * Obtain the aggregated number of bytes written to the network.
+ * @return total number of bytes written.
+ */
+ public abstract long getNetworkBytesWritten();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 055298d..8801985 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -83,9 +83,22 @@ public class SysInfoLinux extends SysInfo {
"[ \t]*([0-9]*)[ \t]*([0-9]*)[ \t].*");
private CpuTimeTracker cpuTimeTracker;
+ /**
+ * Pattern for parsing /proc/net/dev.
+ */
+ private static final String PROCFS_NETFILE = "/proc/net/dev";
+ private static final Pattern PROCFS_NETFILE_FORMAT =
+ Pattern .compile("^[ \t]*([a-zA-Z]+[0-9]*):" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+).*");
+
+
private String procfsMemFile;
private String procfsCpuFile;
private String procfsStatFile;
+ private String procfsNetFile;
private long jiffyLengthInMillis;
private long ramSize = 0;
@@ -98,6 +111,8 @@ public class SysInfoLinux extends SysInfo {
/* number of physical cores on the system. */
private int numCores = 0;
private long cpuFrequency = 0L; // CPU frequency on the system (kHz)
+ private long numNetBytesRead = 0L; // aggregated bytes read from network
+ private long numNetBytesWritten = 0L; // aggregated bytes written to network
private boolean readMemInfoFile = false;
private boolean readCpuInfoFile = false;
@@ -130,7 +145,7 @@ public class SysInfoLinux extends SysInfo {
public SysInfoLinux() {
this(PROCFS_MEMFILE, PROCFS_CPUINFO, PROCFS_STAT,
- JIFFY_LENGTH_IN_MILLIS);
+ PROCFS_NETFILE, JIFFY_LENGTH_IN_MILLIS);
}
/**
@@ -139,16 +154,19 @@ public class SysInfoLinux extends SysInfo {
* @param procfsMemFile fake file for /proc/meminfo
* @param procfsCpuFile fake file for /proc/cpuinfo
* @param procfsStatFile fake file for /proc/stat
+ * @param procfsNetFile fake file for /proc/net/dev
* @param jiffyLengthInMillis fake jiffy length value
*/
@VisibleForTesting
public SysInfoLinux(String procfsMemFile,
String procfsCpuFile,
String procfsStatFile,
+ String procfsNetFile,
long jiffyLengthInMillis) {
this.procfsMemFile = procfsMemFile;
this.procfsCpuFile = procfsCpuFile;
this.procfsStatFile = procfsStatFile;
+ this.procfsNetFile = procfsNetFile;
this.jiffyLengthInMillis = jiffyLengthInMillis;
this.cpuTimeTracker = new CpuTimeTracker(jiffyLengthInMillis);
}
@@ -338,6 +356,61 @@ public class SysInfoLinux extends SysInfo {
}
}
+ /**
+ * Read /proc/net/dev file, parse and calculate amount
+ * of bytes read and written through the network.
+ */
+ private void readProcNetInfoFile() {
+
+ numNetBytesRead = 0L;
+ numNetBytesWritten = 0L;
+
+ // Read "/proc/net/dev" file
+ BufferedReader in;
+ InputStreamReader fReader;
+ try {
+ fReader = new InputStreamReader(
+ new FileInputStream(procfsNetFile), Charset.forName("UTF-8"));
+ in = new BufferedReader(fReader);
+ } catch (FileNotFoundException f) {
+ return;
+ }
+
+ Matcher mat;
+ try {
+ String str = in.readLine();
+ while (str != null) {
+ mat = PROCFS_NETFILE_FORMAT.matcher(str);
+ if (mat.find()) {
+ assert mat.groupCount() >= 16;
+
+ // ignore loopback interfaces
+ if (mat.group(1).equals("lo")) {
+ str = in.readLine();
+ continue;
+ }
+ numNetBytesRead += Long.parseLong(mat.group(2));
+ numNetBytesWritten += Long.parseLong(mat.group(10));
+ }
+ str = in.readLine();
+ }
+ } catch (IOException io) {
+ LOG.warn("Error reading the stream " + io);
+ } finally {
+ // Close the streams
+ try {
+ fReader.close();
+ try {
+ in.close();
+ } catch (IOException i) {
+ LOG.warn("Error closing the stream " + in);
+ }
+ } catch (IOException i) {
+ LOG.warn("Error closing the stream " + fReader);
+ }
+ }
+ }
+
/** {@inheritDoc} */
@Override
public long getPhysicalMemorySize() {
@@ -405,6 +478,20 @@ public class SysInfoLinux extends SysInfo {
return overallCpuUsage;
}
+ /** {@inheritDoc} */
+ @Override
+ public long getNetworkBytesRead() {
+ readProcNetInfoFile();
+ return numNetBytesRead;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getNetworkBytesWritten() {
+ readProcNetInfoFile();
+ return numNetBytesWritten;
+ }
+
/**
* Test the {@link SysInfoLinux}.
*
@@ -424,6 +511,10 @@ public class SysInfoLinux extends SysInfo {
System.out.println("CPU frequency (kHz) : " + plugin.getCpuFrequency());
System.out.println("Cumulative CPU time (ms) : " +
plugin.getCumulativeCpuTime());
+ System.out.println("Total network read (bytes) : "
+ + plugin.getNetworkBytesRead());
+ System.out.println("Total network written (bytes) : "
+ + plugin.getNetworkBytesWritten());
try {
// Sleep so we can compute the CPU usage
Thread.sleep(500L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
index da4c1c5..f8542a3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
@@ -178,4 +178,19 @@ public class SysInfoWindows extends SysInfo {
refreshIfNeeded();
return cpuUsage;
}
+
+ /** {@inheritDoc} */
+ @Override
+ public long getNetworkBytesRead() {
+ // TODO unimplemented
+ return 0L;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getNetworkBytesWritten() {
+ // TODO unimplemented
+ return 0L;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index 73edc77..2a31f31 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -44,8 +44,10 @@ public class TestSysInfoLinux {
public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
String procfsCpuFile,
String procfsStatFile,
+ String procfsNetFile,
long jiffyLengthInMillis) {
- super(procfsMemFile, procfsCpuFile, procfsStatFile, jiffyLengthInMillis);
+ super(procfsMemFile, procfsCpuFile, procfsStatFile, procfsNetFile,
+ jiffyLengthInMillis);
}
@Override
long getCurrentTime() {
@@ -61,14 +63,17 @@ public class TestSysInfoLinux {
private static final String FAKE_MEMFILE;
private static final String FAKE_CPUFILE;
private static final String FAKE_STATFILE;
+ private static final String FAKE_NETFILE;
private static final long FAKE_JIFFY_LENGTH = 10L;
static {
int randomNum = (new Random()).nextInt(1000000000);
FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum;
FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
+ FAKE_NETFILE = TEST_ROOT_DIR + File.separator + "NETINFO_" + randomNum;
plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
FAKE_STATFILE,
+ FAKE_NETFILE,
FAKE_JIFFY_LENGTH);
}
static final String MEMINFO_FORMAT =
@@ -141,6 +146,17 @@ public class TestSysInfoLinux {
"procs_running 1\n" +
"procs_blocked 0\n";
+ static final String NETINFO_FORMAT =
+ "Inter-| Receive | Transmit\n"+
+ "face |bytes packets errs drop fifo frame compressed multicast|bytes packets"+
+ "errs drop fifo colls carrier compressed\n"+
+ " lo: 42236310 563003 0 0 0 0 0 0 42236310 563003 " +
+ "0 0 0 0 0 0\n"+
+ " eth0: %d 3452527 0 0 0 0 0 299787 %d 1866280 0 0 " +
+ "0 0 0 0\n"+
+ " eth1: %d 3152521 0 0 0 0 0 219781 %d 1866290 0 0 " +
+ "0 0 0 0\n";
+
/**
* Test parsing /proc/stat and /proc/cpuinfo
* @throws IOException
@@ -320,4 +336,26 @@ public class TestSysInfoLinux {
IOUtils.closeQuietly(fWriter);
}
}
+
+ /**
+ * Test parsing /proc/net/dev
+ * @throws IOException
+ */
+ @Test
+ public void parsingProcNetFile() throws IOException {
+ long numBytesReadIntf1 = 2097172468L;
+ long numBytesWrittenIntf1 = 1355620114L;
+ long numBytesReadIntf2 = 1097172460L;
+ long numBytesWrittenIntf2 = 1055620110L;
+ File tempFile = new File(FAKE_NETFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_NETFILE);
+ fWriter.write(String.format(NETINFO_FORMAT,
+ numBytesReadIntf1, numBytesWrittenIntf1,
+ numBytesReadIntf2, numBytesWrittenIntf2));
+ fWriter.close();
+ assertEquals(plugin.getNetworkBytesRead(), numBytesReadIntf1 + numBytesReadIntf2);
+ assertEquals(plugin.getNetworkBytesWritten(), numBytesWrittenIntf1 + numBytesWrittenIntf2);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
index fd4cb83..b86303b 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
@@ -48,6 +48,12 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
"mapred.tasktracker.cumulativecputime.testing";
/** CPU usage percentage for testing */
public static final String CPU_USAGE = "mapred.tasktracker.cpuusage.testing";
+ /** cumulative number of bytes read over the network */
+ public static final String NETWORK_BYTES_READ =
+ "mapred.tasktracker.networkread.testing";
+ /** cumulative number of bytes written over the network */
+ public static final String NETWORK_BYTES_WRITTEN =
+ "mapred.tasktracker.networkwritten.testing";
/** process cumulative CPU usage time for testing */
public static final String PROC_CUMULATIVE_CPU_TIME =
"mapred.tasktracker.proccumulativecputime.testing";
@@ -111,4 +117,17 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
public float getCpuUsage() {
return getConf().getFloat(CPU_USAGE, -1);
}
+
+ /** {@inheritDoc} */
+ @Override
+ public long getNetworkBytesRead() {
+ return getConf().getLong(NETWORK_BYTES_READ, -1);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getNetworkBytesWritten() {
+ return getConf().getLong(NETWORK_BYTES_WRITTEN, -1);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a0752d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
index 5e5f1b4..21724a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
@@ -124,6 +124,22 @@ public class ResourceCalculatorPlugin extends Configured {
return sys.getCpuUsage();
}
+ /**
+ * Obtain the aggregated number of bytes read over the network.
+ * @return total number of bytes read.
+ */
+ public long getNetworkBytesRead() {
+ return sys.getNetworkBytesRead();
+ }
+
+ /**
+ * Obtain the aggregated number of bytes written to the network.
+ * @return total number of bytes written.
+ */
+ public long getNetworkBytesWritten() {
+ return sys.getNetworkBytesWritten();
+ }
+
/**
* Create the ResourceCalculatorPlugin from the class name and configure it. If
* class name is null, this method will try and return a memory calculator