You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/07/13 20:59:44 UTC
[32/48] hadoop git commit: HADOOP-12180. Move
ResourceCalculatorPlugin from YARN to Common. (Chris Douglas via kasha)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f246e1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java
deleted file mode 100644
index a59d503..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Random;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * A JUnit test to test {@link LinuxResourceCalculatorPlugin}
- * Create the fake /proc/ information and verify the parsing and calculation
- */
-public class TestLinuxResourceCalculatorPlugin {
- /**
- * LinuxResourceCalculatorPlugin with a fake timer
- */
- static class FakeLinuxResourceCalculatorPlugin extends
- LinuxResourceCalculatorPlugin {
-
- long currentTime = 0;
- public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
- String procfsCpuFile,
- String procfsStatFile,
- long jiffyLengthInMillis) {
- super(procfsMemFile, procfsCpuFile, procfsStatFile, jiffyLengthInMillis);
- }
- @Override
- long getCurrentTime() {
- return currentTime;
- }
- public void advanceTime(long adv) {
- currentTime += adv * this.getJiffyLengthInMillis();
- }
- }
- private static final FakeLinuxResourceCalculatorPlugin plugin;
- private static String TEST_ROOT_DIR = new Path(System.getProperty(
- "test.build.data", "/tmp")).toString().replace(' ', '+');
- private static final String FAKE_MEMFILE;
- private static final String FAKE_CPUFILE;
- private static final String FAKE_STATFILE;
- private static final long FAKE_JIFFY_LENGTH = 10L;
- static {
- int randomNum = (new Random()).nextInt(1000000000);
- FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum;
- FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
- FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
- plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
- FAKE_STATFILE,
- FAKE_JIFFY_LENGTH);
- }
- static final String MEMINFO_FORMAT =
- "MemTotal: %d kB\n" +
- "MemFree: %d kB\n" +
- "Buffers: 138244 kB\n" +
- "Cached: 947780 kB\n" +
- "SwapCached: 142880 kB\n" +
- "Active: 3229888 kB\n" +
- "Inactive: %d kB\n" +
- "SwapTotal: %d kB\n" +
- "SwapFree: %d kB\n" +
- "Dirty: 122012 kB\n" +
- "Writeback: 0 kB\n" +
- "AnonPages: 2710792 kB\n" +
- "Mapped: 24740 kB\n" +
- "Slab: 132528 kB\n" +
- "SReclaimable: 105096 kB\n" +
- "SUnreclaim: 27432 kB\n" +
- "PageTables: 11448 kB\n" +
- "NFS_Unstable: 0 kB\n" +
- "Bounce: 0 kB\n" +
- "CommitLimit: 4125904 kB\n" +
- "Committed_AS: 4143556 kB\n" +
- "VmallocTotal: 34359738367 kB\n" +
- "VmallocUsed: 1632 kB\n" +
- "VmallocChunk: 34359736375 kB\n" +
- "HugePages_Total: 0\n" +
- "HugePages_Free: 0\n" +
- "HugePages_Rsvd: 0\n" +
- "Hugepagesize: 2048 kB";
-
- static final String CPUINFO_FORMAT =
- "processor : %s\n" +
- "vendor_id : AuthenticAMD\n" +
- "cpu family : 15\n" +
- "model : 33\n" +
- "model name : Dual Core AMD Opteron(tm) Processor 280\n" +
- "stepping : 2\n" +
- "cpu MHz : %f\n" +
- "cache size : 1024 KB\n" +
- "physical id : %s\n" +
- "siblings : 2\n" +
- "core id : %s\n" +
- "cpu cores : 2\n" +
- "fpu : yes\n" +
- "fpu_exception : yes\n" +
- "cpuid level : 1\n" +
- "wp : yes\n" +
- "flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov " +
- "pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt lm " +
- "3dnowext 3dnow pni lahf_lm cmp_legacy\n" +
- "bogomips : 4792.41\n" +
- "TLB size : 1024 4K pages\n" +
- "clflush size : 64\n" +
- "cache_alignment : 64\n" +
- "address sizes : 40 bits physical, 48 bits virtual\n" +
- "power management: ts fid vid ttp";
-
- static final String STAT_FILE_FORMAT =
- "cpu %d %d %d 1646495089 831319 48713 164346 0\n" +
- "cpu0 15096055 30805 3823005 411456015 206027 13 14269 0\n" +
- "cpu1 14760561 89890 6432036 408707910 456857 48074 130857 0\n" +
- "cpu2 12761169 20842 3758639 413976772 98028 411 10288 0\n" +
- "cpu3 12355207 47322 5789691 412354390 70406 213 8931 0\n" +
- "intr 114648668 20010764 2 0 945665 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0\n" +
- "ctxt 242017731764\n" +
- "btime 1257808753\n" +
- "processes 26414943\n" +
- "procs_running 1\n" +
- "procs_blocked 0\n";
-
- /**
- * Test parsing /proc/stat and /proc/cpuinfo
- * @throws IOException
- */
- @Test
- public void parsingProcStatAndCpuFile() throws IOException {
- // Write fake /proc/cpuinfo file.
- long numProcessors = 8;
- long cpuFrequencyKHz = 2392781;
- String fileContent = "";
- for (int i = 0; i < numProcessors; i++) {
- fileContent +=
- String.format(CPUINFO_FORMAT, i, cpuFrequencyKHz / 1000D, 0, 0)
- + "\n";
- }
- File tempFile = new File(FAKE_CPUFILE);
- tempFile.deleteOnExit();
- FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
- fWriter.write(fileContent);
- fWriter.close();
- assertEquals(plugin.getNumProcessors(), numProcessors);
- assertEquals(plugin.getCpuFrequency(), cpuFrequencyKHz);
-
- // Write fake /proc/stat file.
- long uTime = 54972994;
- long nTime = 188860;
- long sTime = 19803373;
- tempFile = new File(FAKE_STATFILE);
- tempFile.deleteOnExit();
- updateStatFile(uTime, nTime, sTime);
- assertEquals(plugin.getCumulativeCpuTime(),
- FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
- assertEquals(plugin.getCpuUsage(), (float)(CpuTimeTracker.UNAVAILABLE),0.0);
-
- // Advance the time and sample again to test the CPU usage calculation
- uTime += 100L;
- plugin.advanceTime(200L);
- updateStatFile(uTime, nTime, sTime);
- assertEquals(plugin.getCumulativeCpuTime(),
- FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
- assertEquals(plugin.getCpuUsage(), 6.25F, 0.0);
-
- // Advance the time and sample again. This time, we call getCpuUsage() only.
- uTime += 600L;
- plugin.advanceTime(300L);
- updateStatFile(uTime, nTime, sTime);
- assertEquals(plugin.getCpuUsage(), 25F, 0.0);
-
- // Advance very short period of time (one jiffy length).
- // In this case, CPU usage should not be updated.
- uTime += 1L;
- plugin.advanceTime(1L);
- updateStatFile(uTime, nTime, sTime);
- assertEquals(plugin.getCumulativeCpuTime(),
- FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
- assertEquals(plugin.getCpuUsage(), 25F, 0.0); // CPU usage is not updated.
- }
-
- /**
- * Write information to fake /proc/stat file
- */
- private void updateStatFile(long uTime, long nTime, long sTime)
- throws IOException {
- FileWriter fWriter = new FileWriter(FAKE_STATFILE);
- fWriter.write(String.format(STAT_FILE_FORMAT, uTime, nTime, sTime));
- fWriter.close();
- }
-
- /**
- * Test parsing /proc/meminfo
- * @throws IOException
- */
- @Test
- public void parsingProcMemFile() throws IOException {
- long memTotal = 4058864L;
- long memFree = 99632L;
- long inactive = 567732L;
- long swapTotal = 2096472L;
- long swapFree = 1818480L;
- File tempFile = new File(FAKE_MEMFILE);
- tempFile.deleteOnExit();
- FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
- fWriter.write(String.format(MEMINFO_FORMAT,
- memTotal, memFree, inactive, swapTotal, swapFree));
-
- fWriter.close();
- assertEquals(plugin.getAvailablePhysicalMemorySize(),
- 1024L * (memFree + inactive));
- assertEquals(plugin.getAvailableVirtualMemorySize(),
- 1024L * (memFree + inactive + swapFree));
- assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
- assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
- }
-
- @Test
- public void testCoreCounts() throws IOException {
-
- String fileContent = "";
- // single core, hyper threading
- long numProcessors = 2;
- long cpuFrequencyKHz = 2392781;
- for (int i = 0; i < numProcessors; i++) {
- fileContent =
- fileContent.concat(String.format(CPUINFO_FORMAT, i,
- cpuFrequencyKHz / 1000D, 0, 0));
- fileContent = fileContent.concat("\n");
- }
- writeFakeCPUInfoFile(fileContent);
- plugin.setReadCpuInfoFile(false);
- assertEquals(numProcessors, plugin.getNumProcessors());
- assertEquals(1, plugin.getNumCores());
-
- // single socket quad core, no hyper threading
- fileContent = "";
- numProcessors = 4;
- for (int i = 0; i < numProcessors; i++) {
- fileContent =
- fileContent.concat(String.format(CPUINFO_FORMAT, i,
- cpuFrequencyKHz / 1000D, 0, i));
- fileContent = fileContent.concat("\n");
- }
- writeFakeCPUInfoFile(fileContent);
- plugin.setReadCpuInfoFile(false);
- assertEquals(numProcessors, plugin.getNumProcessors());
- assertEquals(4, plugin.getNumCores());
-
- // dual socket single core, hyper threading
- fileContent = "";
- numProcessors = 4;
- for (int i = 0; i < numProcessors; i++) {
- fileContent =
- fileContent.concat(String.format(CPUINFO_FORMAT, i,
- cpuFrequencyKHz / 1000D, i / 2, 0));
- fileContent = fileContent.concat("\n");
- }
- writeFakeCPUInfoFile(fileContent);
- plugin.setReadCpuInfoFile(false);
- assertEquals(numProcessors, plugin.getNumProcessors());
- assertEquals(2, plugin.getNumCores());
-
- // dual socket, dual core, no hyper threading
- fileContent = "";
- numProcessors = 4;
- for (int i = 0; i < numProcessors; i++) {
- fileContent =
- fileContent.concat(String.format(CPUINFO_FORMAT, i,
- cpuFrequencyKHz / 1000D, i / 2, i % 2));
- fileContent = fileContent.concat("\n");
- }
- writeFakeCPUInfoFile(fileContent);
- plugin.setReadCpuInfoFile(false);
- assertEquals(numProcessors, plugin.getNumProcessors());
- assertEquals(4, plugin.getNumCores());
-
- // dual socket, dual core, hyper threading
- fileContent = "";
- numProcessors = 8;
- for (int i = 0; i < numProcessors; i++) {
- fileContent =
- fileContent.concat(String.format(CPUINFO_FORMAT, i,
- cpuFrequencyKHz / 1000D, i / 4, (i % 4) / 2));
- fileContent = fileContent.concat("\n");
- }
- writeFakeCPUInfoFile(fileContent);
- plugin.setReadCpuInfoFile(false);
- assertEquals(numProcessors, plugin.getNumProcessors());
- assertEquals(4, plugin.getNumCores());
- }
-
- private void writeFakeCPUInfoFile(String content) throws IOException {
- File tempFile = new File(FAKE_CPUFILE);
- FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
- tempFile.deleteOnExit();
- try {
- fWriter.write(content);
- } finally {
- IOUtils.closeQuietly(fWriter);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f246e1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
index 777ea9f..7a3e0e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
@@ -65,7 +65,7 @@ public class TestResourceCalculatorProcessTree {
@Override
public float getCpuUsagePercent() {
- return CpuTimeTracker.UNAVAILABLE;
+ return UNAVAILABLE;
}
public boolean checkPidPgrpidForMatch() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f246e1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java
deleted file mode 100644
index a9e20bc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util;
-
-import org.junit.Test;
-import static org.junit.Assert.assertTrue;
-
-public class TestWindowsResourceCalculatorPlugin {
-
-
- class WindowsResourceCalculatorPluginTester extends WindowsResourceCalculatorPlugin {
- private String infoStr = null;
- @Override
- String getSystemInfoInfoFromShell() {
- return infoStr;
- }
- }
-
- @Test (timeout = 30000)
- public void parseSystemInfoString() {
- WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
- // info str derived from windows shell command has \r\n termination
- tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n";
- // call a method to refresh values
- tester.getAvailablePhysicalMemorySize();
- // verify information has been refreshed
- assertTrue(tester.vmemSize == 17177038848L);
- assertTrue(tester.memSize == 8589467648L);
- assertTrue(tester.vmemAvailable == 15232745472L);
- assertTrue(tester.memAvailable == 6400417792L);
- assertTrue(tester.numProcessors == 1);
- assertTrue(tester.cpuFrequencyKhz == 2805000L);
- assertTrue(tester.cumulativeCpuTimeMs == 6261812L);
- assertTrue(tester.cpuUsage == -1);
- }
-
- @Test (timeout = 20000)
- public void refreshAndCpuUsage() throws InterruptedException {
- WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
- // info str derived from windows shell command has \r\n termination
- tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n";
- tester.getAvailablePhysicalMemorySize();
- // verify information has been refreshed
- assertTrue(tester.memAvailable == 6400417792L);
- assertTrue(tester.cpuUsage == -1);
-
- tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6261812\r\n";
- tester.getAvailablePhysicalMemorySize();
- // verify information has not been refreshed
- assertTrue(tester.memAvailable == 6400417792L);
- assertTrue(tester.cpuUsage == -1);
-
- Thread.sleep(1500);
- tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6286812\r\n";
- tester.getAvailablePhysicalMemorySize();
- // verify information has been refreshed
- assertTrue(tester.memAvailable == 5400417792L);
- assertTrue(tester.cpuUsage >= 0.1);
- }
-
- @Test (timeout = 20000)
- public void errorInGetSystemInfo() {
- WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
- // info str derived from windows shell command has \r\n termination
- tester.infoStr = null;
- // call a method to refresh values
- tester.getAvailablePhysicalMemorySize();
- }
-
-}