You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/06 21:22:31 UTC
[01/50] [abbrv] hadoop git commit: YARN-3304. Addendum patch.
Cleaning up ResourceCalculatorProcessTree APIs for public use and removing
inconsistencies in the default values. (Junping Du and Karthik Kambatla via
vinodkv)
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 ae0a11167 -> 8898d14c0
YARN-3304. Addendum patch. Cleaning up ResourceCalculatorProcessTree APIs for public use and removing inconsistencies in the default values. (Junping Du and Karthik Kambatla via vinodkv)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0834fe37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0834fe37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0834fe37
Branch: refs/heads/YARN-2928
Commit: 0834fe372ce55a4fd2b8566cbae95b7b908be71d
Parents: 71b4374
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Mar 31 17:27:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
.../yarn/util/ProcfsBasedProcessTree.java | 82 +++++++++++------
.../util/ResourceCalculatorProcessTree.java | 55 ++++++++++-
.../yarn/util/WindowsBasedProcessTree.java | 24 ++++-
.../yarn/util/TestProcfsBasedProcessTree.java | 96 ++++++++++++++++++--
.../util/TestResourceCalculatorProcessTree.java | 10 ++
.../yarn/util/TestWindowsBasedProcessTree.java | 15 ++-
6 files changed, 236 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index 9996a79..df9d28a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -344,15 +344,23 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
public long getVirtualMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
- if ((p != null) && (p.getAge() > olderThanAge)) {
+ if (p != null) {
if (total == UNAVAILABLE ) {
total = 0;
}
- total += p.getVmem();
+ if (p.getAge() > olderThanAge) {
+ total += p.getVmem();
+ }
}
}
return total;
}
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public long getCumulativeVmem(int olderThanAge) {
+ return getVirtualMemorySize(olderThanAge);
+ }
@Override
public long getRssMemorySize(int olderThanAge) {
@@ -365,13 +373,21 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
boolean isAvailable = false;
long totalPages = 0;
for (ProcessInfo p : processTree.values()) {
- if ((p != null) && (p.getAge() > olderThanAge)) {
- totalPages += p.getRssmemPage();
+ if ((p != null) ) {
+ if (p.getAge() > olderThanAge) {
+ totalPages += p.getRssmemPage();
+ }
isAvailable = true;
}
}
return isAvailable ? totalPages * PAGE_SIZE : UNAVAILABLE; // convert # pages to byte
}
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public long getCumulativeRssmem(int olderThanAge) {
+ return getRssMemorySize(olderThanAge);
+ }
/**
* Get the resident set size (RSS) memory used by all the processes
@@ -388,36 +404,42 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
private long getSmapBasedRssMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
- if ((p != null) && (p.getAge() > olderThanAge)) {
- ProcessTreeSmapMemInfo procMemInfo = processSMAPTree.get(p.getPid());
- if (procMemInfo != null) {
- for (ProcessSmapMemoryInfo info : procMemInfo.getMemoryInfoList()) {
- // Do not account for r--s or r-xs mappings
- if (info.getPermission().trim()
- .equalsIgnoreCase(READ_ONLY_WITH_SHARED_PERMISSION)
- || info.getPermission().trim()
- .equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) {
- continue;
- }
- if (total == UNAVAILABLE){
- total = 0;
- }
- total +=
- Math.min(info.sharedDirty, info.pss) + info.privateDirty
- + info.privateClean;
- if (LOG.isDebugEnabled()) {
- LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid()
- + ", SharedDirty : " + info.sharedDirty + ", PSS : "
- + info.pss + ", Private_Dirty : " + info.privateDirty
- + ", Private_Clean : " + info.privateClean + ", total : "
- + (total * KB_TO_BYTES));
+ if (p != null) {
+ // set resource to 0 instead of UNAVAILABLE
+ if (total == UNAVAILABLE){
+ total = 0;
+ }
+ if (p.getAge() > olderThanAge) {
+ ProcessTreeSmapMemInfo procMemInfo = processSMAPTree.get(p.getPid());
+ if (procMemInfo != null) {
+ for (ProcessSmapMemoryInfo info : procMemInfo.getMemoryInfoList()) {
+ // Do not account for r--s or r-xs mappings
+ if (info.getPermission().trim()
+ .equalsIgnoreCase(READ_ONLY_WITH_SHARED_PERMISSION)
+ || info.getPermission().trim()
+ .equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) {
+ continue;
+ }
+
+ total +=
+ Math.min(info.sharedDirty, info.pss) + info.privateDirty
+ + info.privateClean;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid()
+ + ", SharedDirty : " + info.sharedDirty + ", PSS : "
+ + info.pss + ", Private_Dirty : " + info.privateDirty
+ + ", Private_Clean : " + info.privateClean + ", total : "
+ + (total * KB_TO_BYTES));
+ }
}
}
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(procMemInfo.toString());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(procMemInfo.toString());
+ }
}
}
+
}
if (total > 0) {
total *= KB_TO_BYTES; // convert to bytes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
index 3c4bf52..01d79d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
@@ -78,6 +78,18 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
public long getVirtualMemorySize() {
return getVirtualMemorySize(0);
}
+
+ /**
+ * Get the virtual memory used by all the processes in the
+ * process-tree.
+ *
+ * @return virtual memory used by the process-tree in bytes,
+ * {@link #UNAVAILABLE} if it cannot be calculated.
+ */
+ @Deprecated
+ public long getCumulativeVmem() {
+ return getCumulativeVmem(0);
+ }
/**
* Get the resident set size (rss) memory used by all the processes
@@ -89,6 +101,18 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
public long getRssMemorySize() {
return getRssMemorySize(0);
}
+
+ /**
+ * Get the resident set size (rss) memory used by all the processes
+ * in the process-tree.
+ *
+ * @return rss memory used by the process-tree in bytes,
+ * {@link #UNAVAILABLE} if it cannot be calculated.
+ */
+ @Deprecated
+ public long getCumulativeRssmem() {
+ return getCumulativeRssmem(0);
+ }
/**
* Get the virtual memory used by all the processes in the
@@ -103,6 +127,21 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
public long getVirtualMemorySize(int olderThanAge) {
return UNAVAILABLE;
}
+
+ /**
+ * Get the virtual memory used by all the processes in the
+ * process-tree that are older than the passed in age.
+ *
+ * @param olderThanAge processes above this age are included in the
+ * memory addition
+ * @return virtual memory used by the process-tree in bytes for
+ * processes older than the specified age, {@link #UNAVAILABLE} if it
+ * cannot be calculated.
+ */
+ @Deprecated
+ public long getCumulativeVmem(int olderThanAge) {
+ return UNAVAILABLE;
+ }
/**
* Get the resident set size (rss) memory used by all the processes
@@ -117,6 +156,21 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
public long getRssMemorySize(int olderThanAge) {
return UNAVAILABLE;
}
+
+ /**
+ * Get the resident set size (rss) memory used by all the processes
+ * in the process-tree that are older than the passed in age.
+ *
+ * @param olderThanAge processes above this age are included in the
+ * memory addition
+ * @return rss memory used by the process-tree in bytes for
+ * processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ * calculated.
+ */
+ @Deprecated
+ public long getCumulativeRssmem(int olderThanAge) {
+ return UNAVAILABLE;
+ }
/**
* Get the CPU time in millisecond used by all the processes in the
@@ -158,7 +212,6 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
* @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
* is not available for this system.
*/
- @Private
public static ResourceCalculatorProcessTree getResourceCalculatorProcessTree(
String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
index 90426be..7d9c7d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
@@ -176,29 +176,45 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
public long getVirtualMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
- if ((p != null) && (p.age > olderThanAge)) {
+ if (p != null) {
if (total == UNAVAILABLE) {
total = 0;
}
- total += p.vmem;
+ if (p.age > olderThanAge) {
+ total += p.vmem;
+ }
}
}
return total;
}
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public long getCumulativeVmem(int olderThanAge) {
+ return getVirtualMemorySize(olderThanAge);
+ }
@Override
public long getRssMemorySize(int olderThanAge) {
long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
- if ((p != null) && (p.age > olderThanAge)) {
+ if (p != null) {
if (total == UNAVAILABLE) {
total = 0;
}
- total += p.workingSet;
+ if (p.age > olderThanAge) {
+ total += p.workingSet;
+ }
}
}
return total;
}
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public long getCumulativeRssmem(int olderThanAge) {
+ return getRssMemorySize(olderThanAge);
+ }
@Override
public long getCumulativeCpuTime() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
index eeeeb52..59cef07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
@@ -117,6 +117,7 @@ public class TestProcfsBasedProcessTree {
}
@Test(timeout = 30000)
+ @SuppressWarnings("deprecation")
public void testProcessTree() throws Exception {
try {
Assert.assertTrue(ProcfsBasedProcessTree.isAvailable());
@@ -226,9 +227,13 @@ public class TestProcfsBasedProcessTree {
// ProcessTree is gone now. Any further calls should be sane.
p.updateProcessTree();
Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
+
Assert.assertTrue(
"vmem for the gone-process is " + p.getVirtualMemorySize()
+ " . It should be zero.", p.getVirtualMemorySize() == 0);
+ Assert.assertTrue(
+ "vmem (old API) for the gone-process is " + p.getCumulativeVmem()
+ + " . It should be zero.", p.getCumulativeVmem() == 0);
Assert.assertTrue(p.toString().equals("[ ]"));
}
@@ -385,6 +390,7 @@ public class TestProcfsBasedProcessTree {
* files.
*/
@Test(timeout = 30000)
+ @SuppressWarnings("deprecation")
public void testCpuAndMemoryForProcessTree() throws IOException {
// test processes
@@ -437,9 +443,13 @@ public class TestProcfsBasedProcessTree {
// verify rss memory
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
- ? 600L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
+ ? 600L * ProcfsBasedProcessTree.PAGE_SIZE :
+ ResourceCalculatorProcessTree.UNAVAILABLE;
Assert.assertEquals("rss memory does not match", cumuRssMem,
processTree.getRssMemorySize());
+ // verify old API
+ Assert.assertEquals("rss memory (old API) does not match", cumuRssMem,
+ processTree.getCumulativeRssmem());
// verify cumulative cpu time
long cumuCpuTime =
@@ -459,6 +469,9 @@ public class TestProcfsBasedProcessTree {
// r--s)
Assert.assertEquals("rss memory does not match",
(100 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
+ // verify old API
+ Assert.assertEquals("rss memory (old API) does not match",
+ (100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem());
// test the cpu time again to see if it cumulates
procInfos[0] =
@@ -524,6 +537,7 @@ public class TestProcfsBasedProcessTree {
testMemForOlderProcesses(true);
}
+ @SuppressWarnings("deprecation")
private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
// initial list of processes
String[] pids = { "100", "200", "300", "400" };
@@ -565,8 +579,12 @@ public class TestProcfsBasedProcessTree {
setSmapsInProceTree(processTree, smapEnabled);
// verify virtual memory
- Assert.assertEquals("Cumulative memory does not match", 700000L,
+ Assert.assertEquals("Virtual memory does not match", 700000L,
processTree.getVirtualMemorySize());
+
+ Assert.assertEquals("Virtual memory (old API) does not match", 700000L,
+ processTree.getCumulativeVmem());
+
// write one more process as child of 100.
String[] newPids = { "500" };
setupPidDirs(procfsRootDir, newPids);
@@ -584,32 +602,58 @@ public class TestProcfsBasedProcessTree {
processTree.updateProcessTree();
Assert.assertEquals("vmem does not include new process",
1200000L, processTree.getVirtualMemorySize());
+
+ Assert.assertEquals("vmem (old API) does not include new process",
+ 1200000L, processTree.getCumulativeVmem());
+
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
- ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
+ ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE :
+ ResourceCalculatorProcessTree.UNAVAILABLE;
Assert.assertEquals("rssmem does not include new process",
cumuRssMem, processTree.getRssMemorySize());
+ // verify old API
+ Assert.assertEquals("rssmem (old API) does not include new process",
+ cumuRssMem, processTree.getCumulativeRssmem());
} else {
Assert.assertEquals("rssmem does not include new process",
100 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
+ // verify old API
+ Assert.assertEquals("rssmem (old API) does not include new process",
+ 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem());
}
// however processes older than 1 iteration will retain the older value
Assert.assertEquals(
"vmem shouldn't have included new process", 700000L,
processTree.getVirtualMemorySize(1));
+ // verify old API
+ Assert.assertEquals(
+ "vmem (old API) shouldn't have included new process", 700000L,
+ processTree.getCumulativeVmem(1));
+
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
- ? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
+ ? 700L * ProcfsBasedProcessTree.PAGE_SIZE :
+ ResourceCalculatorProcessTree.UNAVAILABLE;
Assert.assertEquals(
"rssmem shouldn't have included new process", cumuRssMem,
processTree.getRssMemorySize(1));
+ // Verify old API
+ Assert.assertEquals(
+ "rssmem (old API) shouldn't have included new process", cumuRssMem,
+ processTree.getCumulativeRssmem(1));
+
} else {
Assert.assertEquals(
"rssmem shouldn't have included new process",
100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
+ // Verify old API
+ Assert.assertEquals(
+ "rssmem (old API) shouldn't have included new process",
+ 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1));
}
// one more process
@@ -632,17 +676,32 @@ public class TestProcfsBasedProcessTree {
Assert.assertEquals(
"vmem shouldn't have included new processes", 700000L,
processTree.getVirtualMemorySize(2));
+
+ // verify old API
+ Assert.assertEquals(
+ "vmem (old API) shouldn't have included new processes", 700000L,
+ processTree.getCumulativeVmem(2));
+
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
- ? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
+ ? 700L * ProcfsBasedProcessTree.PAGE_SIZE :
+ ResourceCalculatorProcessTree.UNAVAILABLE;
Assert.assertEquals(
"rssmem shouldn't have included new processes",
cumuRssMem, processTree.getRssMemorySize(2));
+ // Verify old API
+ Assert.assertEquals(
+ "rssmem (old API) shouldn't have included new processes",
+ cumuRssMem, processTree.getCumulativeRssmem(2));
} else {
Assert.assertEquals(
"rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
+ // Verify old API
+ Assert.assertEquals(
+ "rssmem (old API) shouldn't have included new processes",
+ 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2));
}
// processes older than 1 iteration should not include new process,
@@ -650,29 +709,46 @@ public class TestProcfsBasedProcessTree {
Assert.assertEquals(
"vmem shouldn't have included new processes", 1200000L,
processTree.getVirtualMemorySize(1));
+ // verify old API
+ Assert.assertEquals(
+ "vmem (old API) shouldn't have included new processes", 1200000L,
+ processTree.getCumulativeVmem(1));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
- ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
+ ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE :
+ ResourceCalculatorProcessTree.UNAVAILABLE;
Assert.assertEquals(
"rssmem shouldn't have included new processes",
cumuRssMem, processTree.getRssMemorySize(1));
+ // verify old API
+ Assert.assertEquals(
+ "rssmem (old API) shouldn't have included new processes",
+ cumuRssMem, processTree.getCumulativeRssmem(1));
} else {
Assert.assertEquals(
"rssmem shouldn't have included new processes",
100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
+ Assert.assertEquals(
+ "rssmem (old API) shouldn't have included new processes",
+ 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1));
}
// no processes older than 3 iterations
Assert.assertEquals(
"Getting non-zero vmem for processes older than 3 iterations",
- UNAVAILABLE, processTree.getVirtualMemorySize(3));
+ 0, processTree.getVirtualMemorySize(3));
+ // verify old API
Assert.assertEquals(
- "Getting non-zero rssmem for processes older than 3 iterations",
- UNAVAILABLE, processTree.getRssMemorySize(3));
+ "Getting non-zero vmem (old API) for processes older than 3 iterations",
+ 0, processTree.getCumulativeVmem(3));
Assert.assertEquals(
"Getting non-zero rssmem for processes older than 3 iterations",
- UNAVAILABLE, processTree.getRssMemorySize(3));
+ 0, processTree.getRssMemorySize(3));
+ // verify old API
+ Assert.assertEquals(
+ "Getting non-zero rssmem (old API) for processes older than 3 iterations",
+ 0, processTree.getCumulativeRssmem(3));
} finally {
FileUtil.fullyDelete(procfsRootDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
index 9bf525c..777ea9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
@@ -44,10 +44,20 @@ public class TestResourceCalculatorProcessTree {
public long getRssMemorySize(int age) {
return 0;
}
+
+ @SuppressWarnings("deprecation")
+ public long getCumulativeRssmem(int age) {
+ return 0;
+ }
public long getVirtualMemorySize(int age) {
return 0;
}
+
+ @SuppressWarnings("deprecation")
+ public long getCumulativeVmem(int age) {
+ return 0;
+ }
public long getCumulativeCpuTime() {
return 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0834fe37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
index 2a208a1..80c5b02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
@@ -41,6 +41,7 @@ public class TestWindowsBasedProcessTree {
}
@Test (timeout = 30000)
+ @SuppressWarnings("deprecation")
public void tree() {
if( !Shell.WINDOWS) {
LOG.info("Platform not Windows. Not testing");
@@ -49,30 +50,42 @@ public class TestWindowsBasedProcessTree {
assertTrue("WindowsBasedProcessTree should be available on Windows",
WindowsBasedProcessTree.isAvailable());
-
WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1");
pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getVirtualMemorySize() == 2048);
+ assertTrue(pTree.getCumulativeVmem() == 2048);
assertTrue(pTree.getVirtualMemorySize(0) == 2048);
+ assertTrue(pTree.getCumulativeVmem(0) == 2048);
+
assertTrue(pTree.getRssMemorySize() == 2048);
+ assertTrue(pTree.getCumulativeRssmem() == 2048);
assertTrue(pTree.getRssMemorySize(0) == 2048);
+ assertTrue(pTree.getCumulativeRssmem(0) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 1000);
pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getVirtualMemorySize() == 3072);
+ assertTrue(pTree.getCumulativeVmem() == 3072);
assertTrue(pTree.getVirtualMemorySize(1) == 2048);
+ assertTrue(pTree.getCumulativeVmem(1) == 2048);
assertTrue(pTree.getRssMemorySize() == 3072);
+ assertTrue(pTree.getCumulativeRssmem() == 3072);
assertTrue(pTree.getRssMemorySize(1) == 2048);
+ assertTrue(pTree.getCumulativeRssmem(1) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 3000);
pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getVirtualMemorySize() == 2048);
+ assertTrue(pTree.getCumulativeVmem() == 2048);
assertTrue(pTree.getVirtualMemorySize(2) == 2048);
+ assertTrue(pTree.getCumulativeVmem(2) == 2048);
assertTrue(pTree.getRssMemorySize() == 2048);
+ assertTrue(pTree.getCumulativeRssmem() == 2048);
assertTrue(pTree.getRssMemorySize(2) == 2048);
+ assertTrue(pTree.getCumulativeRssmem(2) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 4000);
}
}
[15/50] [abbrv] hadoop git commit: HDFS-8009. Signal congestion on
the DataNode. Contributed by Haohui Mai.
Posted by zj...@apache.org.
HDFS-8009. Signal congestion on the DataNode. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d7c0758
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d7c0758
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d7c0758
Branch: refs/heads/YARN-2928
Commit: 0d7c0758b7079ed1660f124d2adcc91ccc9e7289
Parents: a920ab0
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Apr 1 10:56:53 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:11 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../apache/hadoop/hdfs/server/datanode/DataNode.java | 12 ++++++++++--
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7c0758/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4247ea6..cba53a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -371,6 +371,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7671. hdfs user guide should point to the common rack awareness doc.
(Kai Sasaki via aajisaka)
+ HDFS-8009. Signal congestion on the DataNode. (wheat9)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7c0758/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 071aba1..50dccb8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -354,6 +354,9 @@ public class DataNode extends ReconfigurableBase
private String dnUserName = null;
private SpanReceiverHost spanReceiverHost;
+ private static final int NUM_CORES = Runtime.getRuntime()
+ .availableProcessors();
+ private static final double CONGESTION_RATIO = 1.5;
/**
* Creates a dummy DataNode for testing purpose.
@@ -484,8 +487,13 @@ public class DataNode extends ReconfigurableBase
* </ul>
*/
public PipelineAck.ECN getECN() {
- return pipelineSupportECN ? PipelineAck.ECN.SUPPORTED : PipelineAck.ECN
- .DISABLED;
+ if (!pipelineSupportECN) {
+ return PipelineAck.ECN.DISABLED;
+ }
+ double load = ManagementFactory.getOperatingSystemMXBean()
+ .getSystemLoadAverage();
+ return load > NUM_CORES * CONGESTION_RATIO ? PipelineAck.ECN.CONGESTED :
+ PipelineAck.ECN.SUPPORTED;
}
/**
[04/50] [abbrv] hadoop git commit: YARN-3412. RM tests should use
MockRM where possible. (kasha)
Posted by zj...@apache.org.
YARN-3412. RM tests should use MockRM where possible. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b9d546c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b9d546c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b9d546c
Branch: refs/heads/YARN-2928
Commit: 5b9d546c479dc41356aad99038003349d810f84c
Parents: 8af286f
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Mar 31 09:14:15 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../yarn/server/resourcemanager/TestMoveApplication.java | 3 +--
.../yarn/server/resourcemanager/TestResourceManager.java | 6 ++----
.../server/resourcemanager/monitor/TestSchedulingMonitor.java | 3 ++-
.../server/resourcemanager/recovery/TestZKRMStateStore.java | 7 +++----
.../resourcemanager/scheduler/fair/TestFairScheduler.java | 3 +--
.../scheduler/fair/TestFairSchedulerEventLog.java | 4 ++--
.../resourcemanager/scheduler/fifo/TestFifoScheduler.java | 5 ++---
8 files changed, 15 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2466642..3cf9847 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -175,6 +175,8 @@ Release 2.8.0 - UNRELEASED
YARN-3400. [JDK 8] Build Failure due to unreported exceptions in
RPCUtil (rkanter)
+ YARN-3412. RM tests should use MockRM where possible. (kasha)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
index 36153de..d2bde80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestMoveApplication.java
@@ -52,8 +52,7 @@ public class TestMoveApplication {
FifoSchedulerWithMove.class);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, " ");
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
- resourceManager = new ResourceManager();
- resourceManager.init(conf);
+ resourceManager = new MockRM(conf);
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
resourceManager.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index 6735575..fbf54fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -212,9 +212,8 @@ public class TestResourceManager {
public void testResourceManagerInitConfigValidation() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, -1);
- resourceManager = new ResourceManager();
try {
- resourceManager.init(conf);
+ resourceManager = new MockRM(conf);
fail("Exception is expected because the global max attempts" +
" is negative.");
} catch (YarnRuntimeException e) {
@@ -229,9 +228,8 @@ public class TestResourceManager {
Configuration conf = new YarnConfiguration();
conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000);
conf.setLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1001);
- resourceManager = new ResourceManager();;
try {
- resourceManager.init(conf);
+ resourceManager = new MockRM(conf);
} catch (YarnRuntimeException e) {
// Exception is expected.
if (!e.getMessage().startsWith("Nodemanager expiry interval should be no"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
index 5bf61e3..9ec17da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
import org.junit.Test;
@@ -35,7 +36,7 @@ public class TestSchedulingMonitor {
conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
- ResourceManager rm = new ResourceManager();
+ ResourceManager rm = new MockRM();
try {
rm.init(conf);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index bbbf5a1..608adbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -224,8 +225,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234);
conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
- ResourceManager rm1 = new ResourceManager();
- rm1.init(conf1);
+ ResourceManager rm1 = new MockRM(conf1);
rm1.start();
rm1.getRMContext().getRMAdminService().transitionToActive(req);
assertEquals("RM with ZKStore didn't start",
@@ -236,8 +236,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678);
conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
- ResourceManager rm2 = new ResourceManager();
- rm2.init(conf2);
+ ResourceManager rm2 = new MockRM(conf2);
rm2.start();
rm2.getRMContext().getRMAdminService().transitionToActive(req);
assertEquals("RM with ZKStore didn't start",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 7600a35..ff215cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -117,8 +117,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void setUp() throws IOException {
scheduler = new FairScheduler();
conf = createConfiguration();
- resourceManager = new ResourceManager();
- resourceManager.init(conf);
+ resourceManager = new MockRM(conf);
// TODO: This test should really be using MockRM. For now starting stuff
// that is needed at a bare minimum.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
index 311b531..a8741d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
import java.io.IOException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -48,8 +49,7 @@ public class TestFairSchedulerEventLog {
// All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
- resourceManager = new ResourceManager();
- resourceManager.init(conf);
+ resourceManager = new MockRM(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
scheduler.init(conf);
scheduler.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b9d546c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 5fd6c92..77eebdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -116,11 +116,10 @@ public class TestFifoScheduler {
@Before
public void setUp() throws Exception {
- resourceManager = new ResourceManager();
conf = new Configuration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
FifoScheduler.class, ResourceScheduler.class);
- resourceManager.init(conf);
+ resourceManager = new MockRM(conf);
}
@After
[29/50] [abbrv] hadoop git commit: YARN-3365. Enhanced NodeManager to
support using the 'tc' tool via container-executor for outbound network
traffic control. Contributed by Sidharta Seethana.
Posted by zj...@apache.org.
YARN-3365. Enhanced NodeManager to support using the 'tc' tool via container-executor for outbound network traffic control. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f66d408
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f66d408
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f66d408
Branch: refs/heads/YARN-2928
Commit: 4f66d40802a81de710ad8b1909abee2e20e1007d
Parents: 503e490
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Apr 2 16:53:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:14 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 +
.../impl/container-executor.c | 100 +++-
.../impl/container-executor.h | 34 +-
.../main/native/container-executor/impl/main.c | 465 +++++++++++++------
.../nodemanager/TestLinuxContainerExecutor.java | 421 +++++++++++------
5 files changed, 722 insertions(+), 302 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 18a3c37..bcd2286 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -112,6 +112,10 @@ Release 2.8.0 - UNRELEASED
YARN-3345. Add non-exclusive node label API. (Wangda Tan via jianhe)
+ YARN-3365. Enhanced NodeManager to support using the 'tc' tool via
+ container-executor for outbound network traffic control. (Sidharta Seethana
+ via vinodkv)
+
IMPROVEMENTS
YARN-1880. Cleanup TestApplicationClientProtocolOnHA
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index edfd25f..485399a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -40,6 +40,12 @@ static const int DEFAULT_MIN_USERID = 1000;
static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0};
+//location of traffic control binary
+static const char* TC_BIN = "/sbin/tc";
+static const char* TC_MODIFY_STATE_OPTS [] = { "-b" , NULL};
+static const char* TC_READ_STATE_OPTS [] = { "-b", NULL};
+static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL};
+
//struct to store the user details
struct passwd *user_detail = NULL;
@@ -291,27 +297,20 @@ static int write_exit_code_file(const char* exit_code_file, int exit_code) {
return 0;
}
-/**
- * Wait for the container process to exit and write the exit code to
- * the exit code file.
- * Returns the exit code of the container process.
- */
-static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) {
+static int wait_and_get_exit_code(pid_t pid) {
int child_status = -1;
int exit_code = -1;
int waitpid_result;
- if (change_effective_user(nm_uid, nm_gid) != 0) {
- return -1;
- }
do {
- waitpid_result = waitpid(pid, &child_status, 0);
+ waitpid_result = waitpid(pid, &child_status, 0);
} while (waitpid_result == -1 && errno == EINTR);
+
if (waitpid_result < 0) {
- fprintf(LOGFILE, "Error waiting for container process %d - %s\n",
- pid, strerror(errno));
+ fprintf(LOGFILE, "error waiting for process %d - %s\n", pid, strerror(errno));
return -1;
}
+
if (WIFEXITED(child_status)) {
exit_code = WEXITSTATUS(child_status);
} else if (WIFSIGNALED(child_status)) {
@@ -319,9 +318,26 @@ static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) {
} else {
fprintf(LOGFILE, "Unable to determine exit status for pid %d\n", pid);
}
+
+ return exit_code;
+}
+
+/**
+ * Wait for the container process to exit and write the exit code to
+ * the exit code file.
+ * Returns the exit code of the container process.
+ */
+static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) {
+ int exit_code = -1;
+
+ if (change_effective_user(nm_uid, nm_gid) != 0) {
+ return -1;
+ }
+ exit_code = wait_and_get_exit_code(pid);
if (write_exit_code_file(exit_code_file, exit_code) < 0) {
return -1;
}
+
return exit_code;
}
@@ -1470,3 +1486,63 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
#endif
}
+static int run_traffic_control(const char *opts[], char *command_file) {
+ const int max_tc_args = 16;
+ char *args[max_tc_args];
+ int i = 0, j = 0;
+
+ args[i++] = TC_BIN;
+ while (opts[j] != NULL && i < max_tc_args - 1) {
+ args[i] = opts[j];
+ ++i, ++j;
+ }
+ //too many args to tc
+ if (i == max_tc_args - 1) {
+ fprintf(LOGFILE, "too many args to tc");
+ return TRAFFIC_CONTROL_EXECUTION_FAILED;
+ }
+ args[i++] = command_file;
+ args[i] = 0;
+
+ pid_t child_pid = fork();
+ if (child_pid != 0) {
+ int exit_code = wait_and_get_exit_code(child_pid);
+ if (exit_code != 0) {
+ fprintf(LOGFILE, "failed to execute tc command!\n");
+ return TRAFFIC_CONTROL_EXECUTION_FAILED;
+ }
+ unlink(command_file);
+ return 0;
+ } else {
+ execv(TC_BIN, args);
+ //if we reach here, exec failed
+ fprintf(LOGFILE, "failed to execute tc command! error: %s\n", strerror(errno));
+ return TRAFFIC_CONTROL_EXECUTION_FAILED;
+ }
+}
+
+/**
+ * Run a batch of tc commands that modify interface configuration. command_file
+ * is deleted after being used.
+ */
+int traffic_control_modify_state(char *command_file) {
+ return run_traffic_control(TC_MODIFY_STATE_OPTS, command_file);
+}
+
+/**
+ * Run a batch of tc commands that read interface configuration. Output is
+ * written to standard output and it is expected to be read and parsed by the
+ * calling process. command_file is deleted after being used.
+ */
+int traffic_control_read_state(char *command_file) {
+ return run_traffic_control(TC_READ_STATE_OPTS, command_file);
+}
+
+/**
+ * Run a batch of tc commands that read interface stats. Output is
+ * written to standard output and it is expected to be read and parsed by the
+ * calling process. command_file is deleted after being used.
+ */
+int traffic_control_read_stats(char *command_file) {
+ return run_traffic_control(TC_READ_STATS_OPTS, command_file);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index b1efd6a..43ef98d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -54,7 +54,20 @@ enum errorcodes {
INVALID_CONFIG_FILE = 24,
SETSID_OPER_FAILED = 25,
WRITE_PIDFILE_FAILED = 26,
- WRITE_CGROUP_FAILED = 27
+ WRITE_CGROUP_FAILED = 27,
+ TRAFFIC_CONTROL_EXECUTION_FAILED = 28
+};
+
+enum operations {
+ CHECK_SETUP = 1,
+ MOUNT_CGROUPS = 2,
+ TRAFFIC_CONTROL_MODIFY_STATE = 3,
+ TRAFFIC_CONTROL_READ_STATE = 4,
+ TRAFFIC_CONTROL_READ_STATS = 5,
+ RUN_AS_USER_INITIALIZE_CONTAINER = 6,
+ RUN_AS_USER_LAUNCH_CONTAINER = 7,
+ RUN_AS_USER_SIGNAL_CONTAINER = 8,
+ RUN_AS_USER_DELETE = 9
};
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@@ -209,3 +222,22 @@ int check_dir(char* npath, mode_t st_mode, mode_t desired,
int create_validate_dir(char* npath, mode_t perm, char* path,
int finalComponent);
+
+/**
+ * Run a batch of tc commands that modify interface configuration
+ */
+int traffic_control_modify_state(char *command_file);
+
+/**
+ * Run a batch of tc commands that read interface configuration. Output is
+ * written to standard output and it is expected to be read and parsed by the
+ * calling process.
+ */
+int traffic_control_read_state(char *command_file);
+
+/**
+ * Run a batch of tc commands that read interface stats. Output is
+ * written to standard output and it is expected to be read and parsed by the
+ * calling process.
+ */
+int traffic_control_read_stats(char *command_file);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 9b5e784..63fbfe4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -42,84 +42,71 @@
#error HADOOP_CONF_DIR must be defined
#endif
-void display_usage(FILE *stream) {
- fprintf(stream,
- "Usage: container-executor --checksetup\n");
- fprintf(stream,
- "Usage: container-executor --mount-cgroups "\
- "hierarchy controller=path...\n");
- fprintf(stream,
- "Usage: container-executor user yarn-user command command-args\n");
- fprintf(stream, "Commands:\n");
- fprintf(stream, " initialize container: %2d appid tokens " \
- "nm-local-dirs nm-log-dirs cmd app...\n", INITIALIZE_CONTAINER);
- fprintf(stream,
- " launch container: %2d appid containerid workdir "\
- "container-script tokens pidfile nm-local-dirs nm-log-dirs resources\n",
- LAUNCH_CONTAINER);
- fprintf(stream, " signal container: %2d container-pid signal\n",
- SIGNAL_CONTAINER);
- fprintf(stream, " delete as user: %2d relative-path\n",
- DELETE_AS_USER);
+static void display_usage(FILE *stream) {
+ char *usage_template =
+ "Usage: container-executor --checksetup\n" \
+ " container-executor --mount-cgroups <hierarchy> <controller=path>...\n" \
+ " container-executor --tc-modify-state <command-file>\n" \
+ " container-executor --tc-read-state <command-file>\n" \
+ " container-executor --tc-read-stats <command-file>\n" \
+ " container-executor <user> <yarn-user> <command> <command-args>\n" \
+ " where command and command-args: \n" \
+ " initialize container: %2d appid tokens nm-local-dirs nm-log-dirs cmd app...\n" \
+ " launch container: %2d appid containerid workdir container-script " \
+ "tokens pidfile nm-local-dirs nm-log-dirs resources optional-tc-command-file\n" \
+ " signal container: %2d container-pid signal\n" \
+ " delete as user: %2d relative-path\n" ;
+
+
+ fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER,
+ SIGNAL_CONTAINER, DELETE_AS_USER);
}
-int main(int argc, char **argv) {
- int invalid_args = 0;
- int do_check_setup = 0;
- int do_mount_cgroups = 0;
-
- LOGFILE = stdout;
- ERRORFILE = stderr;
+/* Sets up log files for normal/error logging */
+static void open_log_files() {
+ if (LOGFILE == NULL) {
+ LOGFILE = stdout;
+ }
- if (argc > 1) {
- if (strcmp("--mount-cgroups", argv[1]) == 0) {
- do_mount_cgroups = 1;
- }
+ if (ERRORFILE == NULL) {
+ ERRORFILE = stderr;
}
+}
- // Minimum number of arguments required to run
- // the std. container-executor commands is 4
- // 4 args not needed for checksetup option
- if (argc < 4 && !do_mount_cgroups) {
- invalid_args = 1;
- if (argc == 2) {
- const char *arg1 = argv[1];
- if (strcmp("--checksetup", arg1) == 0) {
- invalid_args = 0;
- do_check_setup = 1;
- }
- }
+/* Flushes and closes log files */
+static void flush_and_close_log_files() {
+ if (LOGFILE != NULL) {
+ fflush(LOGFILE);
+ fclose(LOGFILE);
+ LOGFILE = NULL;
}
- if (invalid_args != 0) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+if (ERRORFILE != NULL) {
+ fflush(ERRORFILE);
+ fclose(ERRORFILE);
+ ERRORFILE = NULL;
}
+}
- int command;
- const char * app_id = NULL;
- const char * container_id = NULL;
- const char * cred_file = NULL;
- const char * script_file = NULL;
- const char * current_dir = NULL;
- const char * pid_file = NULL;
-
- int exit_code = 0;
-
- char * dir_to_be_deleted = NULL;
+/** Validates the current container-executor setup. Causes program exit
+in case of validation failures. Also sets up configuration / group information etc.,
+This function is to be called in every invocation of container-executor, irrespective
+of whether an explicit checksetup operation is requested. */
+static void assert_valid_setup(char *current_executable) {
char *executable_file = get_executable();
char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
- char *conf_file = resolve_config_path(orig_conf_file, argv[0]);
- char *local_dirs, *log_dirs;
- char *resources, *resources_key, *resources_value;
+ char *conf_file = resolve_config_path(orig_conf_file, current_executable);
if (conf_file == NULL) {
fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
+ flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
+
if (check_configuration_permissions(conf_file) != 0) {
+ flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
read_config(conf_file);
@@ -129,13 +116,14 @@ int main(int argc, char **argv) {
char *nm_group = get_value(NM_GROUP_KEY);
if (nm_group == NULL) {
fprintf(ERRORFILE, "Can't get configured value for %s.\n", NM_GROUP_KEY);
+ flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
struct group *group_info = getgrnam(nm_group);
if (group_info == NULL) {
fprintf(ERRORFILE, "Can't get group information for %s - %s.\n", nm_group,
strerror(errno));
- fflush(LOGFILE);
+ flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
set_nm_uid(getuid(), group_info->gr_gid);
@@ -146,91 +134,162 @@ int main(int argc, char **argv) {
if (check_executor_permissions(executable_file) != 0) {
fprintf(ERRORFILE, "Invalid permissions on container-executor binary.\n");
- return INVALID_CONTAINER_EXEC_PERMISSIONS;
+ flush_and_close_log_files();
+ exit(INVALID_CONTAINER_EXEC_PERMISSIONS);
}
+}
+
+
+/* Use to store parsed input parmeters for various operations */
+static struct {
+ char *cgroups_hierarchy;
+ char *traffic_control_command_file;
+ const char * run_as_user_name;
+ const char * yarn_user_name;
+ char *local_dirs;
+ char *log_dirs;
+ char *resources_key;
+ char *resources_value;
+ char **resources_values;
+ const char * app_id;
+ const char * container_id;
+ const char * cred_file;
+ const char * script_file;
+ const char * current_dir;
+ const char * pid_file;
+ const char *dir_to_be_deleted;
+ int container_pid;
+ int signal;
+} cmd_input;
+
+static int validate_run_as_user_commands(int argc, char **argv, int *operation);
+
+/* Validates that arguments used in the invocation are valid. In case of validation
+failure, an 'errorcode' is returned. In case of successful validation, a zero is
+returned and 'operation' is populated based on the operation being requested.
+Ideally, we should re-factor container-executor to use a more structured, command
+line parsing mechanism (e.g getopt). For the time being, we'll use this manual
+validation mechanism so that we don't have to change the invocation interface.
+*/
- if (do_check_setup != 0) {
- // basic setup checks done
- // verified configs available and valid
- // verified executor permissions
+static int validate_arguments(int argc, char **argv , int *operation) {
+ if (argc < 2) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+
+ if (strcmp("--checksetup", argv[1]) == 0) {
+ *operation = CHECK_SETUP;
return 0;
}
- if (do_mount_cgroups) {
+ if (strcmp("--mount-cgroups", argv[1]) == 0) {
+ if (argc < 4) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
optind++;
- char *hierarchy = argv[optind++];
- int result = 0;
+ cmd_input.cgroups_hierarchy = argv[optind++];
+ *operation = MOUNT_CGROUPS;
+ return 0;
+ }
- while (optind < argc && result == 0) {
- result = mount_cgroup(argv[optind++], hierarchy);
+ if (strcmp("--tc-modify-state", argv[1]) == 0) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
}
-
- return result;
+ optind++;
+ cmd_input.traffic_control_command_file = argv[optind++];
+ *operation = TRAFFIC_CONTROL_MODIFY_STATE;
+ return 0;
}
- //checks done for user name
- if (argv[optind] == NULL) {
- fprintf(ERRORFILE, "Invalid user name.\n");
- return INVALID_USER_NAME;
+ if (strcmp("--tc-read-state", argv[1]) == 0) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.traffic_control_command_file = argv[optind++];
+ *operation = TRAFFIC_CONTROL_READ_STATE;
+ return 0;
}
- int ret = set_user(argv[optind]);
- if (ret != 0) {
- return ret;
+ if (strcmp("--tc-read-stats", argv[1]) == 0) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.traffic_control_command_file = argv[optind++];
+ *operation = TRAFFIC_CONTROL_READ_STATS;
+ return 0;
}
- // this string is used for building pathnames, the
- // process management is done based on the 'user_detail'
- // global, which was set by 'set_user()' above
- optind = optind + 1;
- char *yarn_user_name = argv[optind];
- if (yarn_user_name == NULL) {
- fprintf(ERRORFILE, "Invalid yarn user name.\n");
- return INVALID_USER_NAME;
+ /* Now we have to validate 'run as user' operations that don't use
+ a 'long option' - we should fix this at some point. The validation/argument
+ parsing here is extensive enough that it done in a separate function */
+
+ return validate_run_as_user_commands(argc, argv, operation);
+}
+
+/* Parse/validate 'run as user' commands */
+static int validate_run_as_user_commands(int argc, char **argv, int *operation) {
+ /* We need at least the following arguments in order to proceed further :
+ <user>, <yarn-user> <command> - i.e at argc should be at least 4 */
+
+ if (argc < 4) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
}
-
- optind = optind + 1;
- command = atoi(argv[optind++]);
- fprintf(LOGFILE, "main : command provided %d\n",command);
- fprintf(LOGFILE, "main : user is %s\n", user_detail->pw_name);
- fprintf(LOGFILE, "main : requested yarn user is %s\n", yarn_user_name);
+ cmd_input.run_as_user_name = argv[optind++];
+ cmd_input.yarn_user_name = argv[optind++];
+ int command = atoi(argv[optind++]);
+
+ fprintf(LOGFILE, "main : command provided %d\n", command);
+ fprintf(LOGFILE, "main : run as user is %s\n", cmd_input.run_as_user_name);
+ fprintf(LOGFILE, "main : requested yarn user is %s\n", cmd_input.yarn_user_name);
fflush(LOGFILE);
switch (command) {
case INITIALIZE_CONTAINER:
if (argc < 9) {
fprintf(ERRORFILE, "Too few arguments (%d vs 9) for initialize container\n",
- argc);
+ argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
- app_id = argv[optind++];
- cred_file = argv[optind++];
- local_dirs = argv[optind++];// good local dirs as a comma separated list
- log_dirs = argv[optind++];// good log dirs as a comma separated list
- exit_code = initialize_app(yarn_user_name, app_id, cred_file,
- extract_values(local_dirs),
- extract_values(log_dirs), argv + optind);
- break;
+ cmd_input.app_id = argv[optind++];
+ cmd_input.cred_file = argv[optind++];
+ cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list
+ cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list
+
+ *operation = RUN_AS_USER_INITIALIZE_CONTAINER;
+ return 0;
+
case LAUNCH_CONTAINER:
- if (argc != 13) {
- fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13) for launch container\n",
- argc);
+ //kill me now.
+ if (!(argc == 13 || argc == 14)) {
+ fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14) for launch container\n",
+ argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
- app_id = argv[optind++];
- container_id = argv[optind++];
- current_dir = argv[optind++];
- script_file = argv[optind++];
- cred_file = argv[optind++];
- pid_file = argv[optind++];
- local_dirs = argv[optind++];// good local dirs as a comma separated list
- log_dirs = argv[optind++];// good log dirs as a comma separated list
- resources = argv[optind++];// key,value pair describing resources
- char *resources_key = malloc(strlen(resources));
- char *resources_value = malloc(strlen(resources));
+
+ cmd_input.app_id = argv[optind++];
+ cmd_input.container_id = argv[optind++];
+ cmd_input.current_dir = argv[optind++];
+ cmd_input.script_file = argv[optind++];
+ cmd_input.cred_file = argv[optind++];
+ cmd_input.pid_file = argv[optind++];
+ cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list
+ cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list
+ char * resources = argv[optind++];// key,value pair describing resources
+ char * resources_key = malloc(strlen(resources));
+ char * resources_value = malloc(strlen(resources));
+
if (get_kv_key(resources, resources_key, strlen(resources)) < 0 ||
get_kv_value(resources, resources_value, strlen(resources)) < 0) {
fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s",
@@ -240,51 +299,157 @@ int main(int argc, char **argv) {
free(resources_value);
return INVALID_ARGUMENT_NUMBER;
}
- char** resources_values = extract_values(resources_value);
- exit_code = launch_container_as_user(yarn_user_name, app_id,
- container_id, current_dir, script_file, cred_file,
- pid_file, extract_values(local_dirs),
- extract_values(log_dirs), resources_key,
- resources_values);
- free(resources_key);
- free(resources_value);
- break;
+
+ //network isolation through tc
+ if (argc == 14) {
+ cmd_input.traffic_control_command_file = argv[optind++];
+ }
+
+ cmd_input.resources_key = resources_key;
+ cmd_input.resources_value = resources_value;
+ cmd_input.resources_values = extract_values(resources_value);
+ *operation = RUN_AS_USER_LAUNCH_CONTAINER;
+ return 0;
+
case SIGNAL_CONTAINER:
if (argc != 6) {
fprintf(ERRORFILE, "Wrong number of arguments (%d vs 6) for " \
"signal container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
- } else {
- char* end_ptr = NULL;
- char* option = argv[optind++];
- int container_pid = strtol(option, &end_ptr, 10);
- if (option == end_ptr || *end_ptr != '\0') {
- fprintf(ERRORFILE, "Illegal argument for container pid %s\n", option);
- fflush(ERRORFILE);
- return INVALID_ARGUMENT_NUMBER;
- }
- option = argv[optind++];
- int signal = strtol(option, &end_ptr, 10);
- if (option == end_ptr || *end_ptr != '\0') {
- fprintf(ERRORFILE, "Illegal argument for signal %s\n", option);
- fflush(ERRORFILE);
- return INVALID_ARGUMENT_NUMBER;
- }
- exit_code = signal_container_as_user(yarn_user_name, container_pid, signal);
}
- break;
+
+ char* end_ptr = NULL;
+ char* option = argv[optind++];
+ cmd_input.container_pid = strtol(option, &end_ptr, 10);
+ if (option == end_ptr || *end_ptr != '\0') {
+ fprintf(ERRORFILE, "Illegal argument for container pid %s\n", option);
+ fflush(ERRORFILE);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ option = argv[optind++];
+ cmd_input.signal = strtol(option, &end_ptr, 10);
+ if (option == end_ptr || *end_ptr != '\0') {
+ fprintf(ERRORFILE, "Illegal argument for signal %s\n", option);
+ fflush(ERRORFILE);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+
+ *operation = RUN_AS_USER_SIGNAL_CONTAINER;
+ return 0;
+
case DELETE_AS_USER:
- dir_to_be_deleted = argv[optind++];
- exit_code= delete_as_user(yarn_user_name, dir_to_be_deleted,
- argv + optind);
- break;
+ cmd_input.dir_to_be_deleted = argv[optind++];
+ *operation = RUN_AS_USER_DELETE;
+ return 0;
default:
fprintf(ERRORFILE, "Invalid command %d not supported.",command);
fflush(ERRORFILE);
- exit_code = INVALID_COMMAND_PROVIDED;
+ return INVALID_COMMAND_PROVIDED;
}
- fclose(LOGFILE);
- fclose(ERRORFILE);
+}
+
+int main(int argc, char **argv) {
+ open_log_files();
+ assert_valid_setup(argv[0]);
+
+ int operation;
+ int ret = validate_arguments(argc, argv, &operation);
+
+ if (ret != 0) {
+ flush_and_close_log_files();
+ return ret;
+ }
+
+ int exit_code = 0;
+
+ switch (operation) {
+ case CHECK_SETUP:
+ //we already did this
+ exit_code = 0;
+ break;
+ case MOUNT_CGROUPS:
+ exit_code = 0;
+
+ while (optind < argc && exit_code == 0) {
+ exit_code = mount_cgroup(argv[optind++], cmd_input.cgroups_hierarchy);
+ }
+
+ break;
+ case TRAFFIC_CONTROL_MODIFY_STATE:
+ exit_code = traffic_control_modify_state(cmd_input.traffic_control_command_file);
+ break;
+ case TRAFFIC_CONTROL_READ_STATE:
+ exit_code = traffic_control_read_state(cmd_input.traffic_control_command_file);
+ break;
+ case TRAFFIC_CONTROL_READ_STATS:
+ exit_code = traffic_control_read_stats(cmd_input.traffic_control_command_file);
+ break;
+ case RUN_AS_USER_INITIALIZE_CONTAINER:
+ exit_code = set_user(cmd_input.run_as_user_name);
+ if (exit_code != 0) {
+ break;
+ }
+
+ exit_code = initialize_app(cmd_input.yarn_user_name,
+ cmd_input.app_id,
+ cmd_input.cred_file,
+ extract_values(cmd_input.local_dirs),
+ extract_values(cmd_input.log_dirs),
+ argv + optind);
+ break;
+ case RUN_AS_USER_LAUNCH_CONTAINER:
+ if (cmd_input.traffic_control_command_file != NULL) {
+ //apply tc rules before switching users and launching the container
+ exit_code = traffic_control_modify_state(cmd_input.traffic_control_command_file);
+ if( exit_code != 0) {
+ //failed to apply tc rules - break out before launching the container
+ break;
+ }
+ }
+
+ exit_code = set_user(cmd_input.run_as_user_name);
+ if (exit_code != 0) {
+ break;
+ }
+
+ exit_code = launch_container_as_user(cmd_input.yarn_user_name,
+ cmd_input.app_id,
+ cmd_input.container_id,
+ cmd_input.current_dir,
+ cmd_input.script_file,
+ cmd_input.cred_file,
+ cmd_input.pid_file,
+ extract_values(cmd_input.local_dirs),
+ extract_values(cmd_input.log_dirs),
+ cmd_input.resources_key,
+ cmd_input.resources_values);
+ free(cmd_input.resources_key);
+ free(cmd_input.resources_value);
+ free(cmd_input.resources_values);
+ break;
+ case RUN_AS_USER_SIGNAL_CONTAINER:
+ exit_code = set_user(cmd_input.run_as_user_name);
+ if (exit_code != 0) {
+ break;
+ }
+
+ exit_code = signal_container_as_user(cmd_input.yarn_user_name,
+ cmd_input.container_pid,
+ cmd_input.signal);
+ break;
+ case RUN_AS_USER_DELETE:
+ exit_code = set_user(cmd_input.run_as_user_name);
+ if (exit_code != 0) {
+ break;
+ }
+
+ exit_code = delete_as_user(cmd_input.yarn_user_name,
+ cmd_input.dir_to_be_deleted,
+ argv + optind);
+ break;
+ }
+
+ flush_and_close_log_files();
return exit_code;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f66d408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index 7417f69..723ac92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -1,20 +1,20 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.yarn.server.nodemanager;
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -32,13 +33,13 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -61,57 +62,88 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
/**
- * This is intended to test the LinuxContainerExecutor code, but because of
- * some security restrictions this can only be done with some special setup
- * first.
- * <br><ol>
+ * This is intended to test the LinuxContainerExecutor code, but because of some
+ * security restrictions this can only be done with some special setup first. <br>
+ * <ol>
* <li>Compile the code with container-executor.conf.dir set to the location you
- * want for testing.
- * <br><pre><code>
+ * want for testing. <br>
+ *
+ * <pre>
+ * <code>
* > mvn clean install -Pnative -Dcontainer-executor.conf.dir=/etc/hadoop
* -DskipTests
- * </code></pre>
+ * </code>
+ * </pre>
*
* <li>Set up <code>${container-executor.conf.dir}/container-executor.cfg</code>
* container-executor.cfg needs to be owned by root and have in it the proper
- * config values.
- * <br><pre><code>
+ * config values. <br>
+ *
+ * <pre>
+ * <code>
* > cat /etc/hadoop/container-executor.cfg
* yarn.nodemanager.linux-container-executor.group=mapred
* #depending on the user id of the application.submitter option
* min.user.id=1
* > sudo chown root:root /etc/hadoop/container-executor.cfg
* > sudo chmod 444 /etc/hadoop/container-executor.cfg
- * </code></pre>
+ * </code>
+ * </pre>
*
- * <li>Move the binary and set proper permissions on it. It needs to be owned
- * by root, the group needs to be the group configured in container-executor.cfg,
+ * <li>Move the binary and set proper permissions on it. It needs to be owned by
+ * root, the group needs to be the group configured in container-executor.cfg,
* and it needs the setuid bit set. (The build will also overwrite it so you
- * need to move it to a place that you can support it.
- * <br><pre><code>
+ * need to move it to a place that you can support it. <br>
+ *
+ * <pre>
+ * <code>
* > cp ./hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/c/container-executor/container-executor /tmp/
* > sudo chown root:mapred /tmp/container-executor
- * > sudo chmod 4550 /tmp/container-executor
- * </code></pre>
+ * > sudo chmod 4050 /tmp/container-executor
+ * </code>
+ * </pre>
*
* <li>Run the tests with the execution enabled (The user you run the tests as
- * needs to be part of the group from the config.
- * <br><pre><code>
+ * needs to be part of the group from the config. <br>
+ *
+ * <pre>
+ * <code>
* mvn test -Dtest=TestLinuxContainerExecutor -Dapplication.submitter=nobody -Dcontainer-executor.path=/tmp/container-executor
- * </code></pre>
+ * </code>
+ * </pre>
+ *
+ * <li>The test suite also contains tests to test mounting of CGroups. By
+ * default, these tests are not run. To run them, add -Dcgroups.mount=<mount-point>
+ * Please note that the test does not unmount the CGroups at the end of the test,
+ * since that requires root permissions. <br>
+ *
+ * <li>The tests that are run are sensitive to directory permissions. All parent
+ * directories must be searchable by the user that the tasks are run as. If you
+ * wish to run the tests in a different directory, please set it using
+ * -Dworkspace.dir
+ *
* </ol>
*/
public class TestLinuxContainerExecutor {
private static final Log LOG = LogFactory
- .getLog(TestLinuxContainerExecutor.class);
-
- private static File workSpace = new File("target",
- TestLinuxContainerExecutor.class.getName() + "-workSpace");
-
+ .getLog(TestLinuxContainerExecutor.class);
+
+ private static File workSpace;
+ static {
+ String basedir = System.getProperty("workspace.dir");
+ if(basedir == null || basedir.isEmpty()) {
+ basedir = "target";
+ }
+ workSpace = new File(basedir,
+ TestLinuxContainerExecutor.class.getName() + "-workSpace");
+ }
+
private LinuxContainerExecutor exec = null;
private String appSubmitter = null;
private LocalDirsHandlerService dirsHandler;
@@ -125,20 +157,26 @@ public class TestLinuxContainerExecutor {
files.mkdir(workSpacePath, null, true);
FileUtil.chmod(workSpace.getAbsolutePath(), "777");
File localDir = new File(workSpace.getAbsoluteFile(), "localDir");
- files.mkdir(new Path(localDir.getAbsolutePath()),
- new FsPermission("777"), false);
+ files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"),
+ false);
File logDir = new File(workSpace.getAbsoluteFile(), "logDir");
- files.mkdir(new Path(logDir.getAbsolutePath()),
- new FsPermission("777"), false);
+ files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"),
+ false);
String exec_path = System.getProperty("container-executor.path");
- if(exec_path != null && !exec_path.isEmpty()) {
+ if (exec_path != null && !exec_path.isEmpty()) {
conf = new Configuration(false);
conf.setClass("fs.AbstractFileSystem.file.impl",
org.apache.hadoop.fs.local.LocalFs.class,
org.apache.hadoop.fs.AbstractFileSystem.class);
- conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, "xuan");
- LOG.info("Setting "+YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
- +"="+exec_path);
+
+ appSubmitter = System.getProperty("application.submitter");
+ if (appSubmitter == null || appSubmitter.isEmpty()) {
+ appSubmitter = "nobody";
+ }
+
+ conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, appSubmitter);
+ LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
+ + "=" + exec_path);
conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
exec = new LinuxContainerExecutor();
exec.setConf(conf);
@@ -146,34 +184,86 @@ public class TestLinuxContainerExecutor {
conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.init(conf);
+ List<String> localDirs = dirsHandler.getLocalDirs();
+ for (String dir : localDirs) {
+ Path userDir = new Path(dir, ContainerLocalizer.USERCACHE);
+ files.mkdir(userDir, new FsPermission("777"), false);
+ // $local/filecache
+ Path fileDir = new Path(dir, ContainerLocalizer.FILECACHE);
+ files.mkdir(fileDir, new FsPermission("777"), false);
+ }
}
- appSubmitter = System.getProperty("application.submitter");
- if(appSubmitter == null || appSubmitter.isEmpty()) {
- appSubmitter = "nobody";
- }
+
}
@After
public void tearDown() throws Exception {
FileContext.getLocalFSFileContext().delete(
- new Path(workSpace.getAbsolutePath()), true);
+ new Path(workSpace.getAbsolutePath()), true);
+ }
+
+ private void cleanupUserAppCache(String user) throws Exception {
+ List<String> localDirs = dirsHandler.getLocalDirs();
+ for (String dir : localDirs) {
+ Path usercachedir = new Path(dir, ContainerLocalizer.USERCACHE);
+ Path userdir = new Path(usercachedir, user);
+ Path appcachedir = new Path(userdir, ContainerLocalizer.APPCACHE);
+ exec.deleteAsUser(user, appcachedir);
+ FileContext.getLocalFSFileContext().delete(usercachedir, true);
+ }
+ }
+
+ private void cleanupUserFileCache(String user) {
+ List<String> localDirs = dirsHandler.getLocalDirs();
+ for (String dir : localDirs) {
+ Path filecache = new Path(dir, ContainerLocalizer.FILECACHE);
+ Path filedir = new Path(filecache, user);
+ exec.deleteAsUser(user, filedir);
+ }
+ }
+
+ private void cleanupLogDirs(String user) {
+ List<String> logDirs = dirsHandler.getLogDirs();
+ for (String dir : logDirs) {
+ String appId = "APP_" + id;
+ String containerId = "CONTAINER_" + (id - 1);
+ Path appdir = new Path(dir, appId);
+ Path containerdir = new Path(appdir, containerId);
+ exec.deleteAsUser(user, containerdir);
+ }
+ }
+
+ private void cleanupAppFiles(String user) throws Exception {
+ cleanupUserAppCache(user);
+ cleanupUserFileCache(user);
+ cleanupLogDirs(user);
+
+ String[] files =
+ { "launch_container.sh", "container_tokens", "touch-file" };
+ Path ws = new Path(workSpace.toURI());
+ for (String file : files) {
+ File f = new File(workSpace, file);
+ if (f.exists()) {
+ exec.deleteAsUser(user, new Path(file), ws);
+ }
+ }
}
private boolean shouldRun() {
- if(exec == null) {
+ if (exec == null) {
LOG.warn("Not running test because container-executor.path is not set");
return false;
}
return true;
}
-
- private String writeScriptFile(String ... cmd) throws IOException {
+
+ private String writeScriptFile(String... cmd) throws IOException {
File f = File.createTempFile("TestLinuxContainerExecutor", ".sh");
f.deleteOnExit();
PrintWriter p = new PrintWriter(new FileOutputStream(f));
p.println("#!/bin/sh");
p.print("exec");
- for(String part: cmd) {
+ for (String part : cmd) {
p.print(" '");
p.print(part.replace("\\", "\\\\").replace("'", "\\'"));
p.print("'");
@@ -182,36 +272,36 @@ public class TestLinuxContainerExecutor {
p.close();
return f.getAbsolutePath();
}
-
+
private int id = 0;
+
private synchronized int getNextId() {
id += 1;
return id;
}
-
+
private ContainerId getNextContainerId() {
ContainerId cId = mock(ContainerId.class);
- String id = "CONTAINER_"+getNextId();
+ String id = "CONTAINER_" + getNextId();
when(cId.toString()).thenReturn(id);
return cId;
}
-
- private int runAndBlock(String ... cmd) throws IOException {
+ private int runAndBlock(String... cmd) throws IOException {
return runAndBlock(getNextContainerId(), cmd);
}
-
- private int runAndBlock(ContainerId cId, String ... cmd) throws IOException {
- String appId = "APP_"+getNextId();
+
+ private int runAndBlock(ContainerId cId, String... cmd) throws IOException {
+ String appId = "APP_" + getNextId();
Container container = mock(Container.class);
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
- HashMap<String, String> env = new HashMap<String,String>();
+ HashMap<String, String> env = new HashMap<String, String>();
when(container.getContainerId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(context.getEnvironment()).thenReturn(env);
-
+
String script = writeScriptFile(cmd);
Path scriptPath = new Path(script);
@@ -221,46 +311,36 @@ public class TestLinuxContainerExecutor {
exec.activateContainer(cId, pidFile);
return exec.launchContainer(container, scriptPath, tokensPath,
- appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
- dirsHandler.getLogDirs());
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
}
-
+
@Test
public void testContainerLocalizer() throws Exception {
- if (!shouldRun()) {
- return;
- }
- List<String> localDirs = dirsHandler.getLocalDirs();
- List<String> logDirs = dirsHandler.getLogDirs();
- for (String localDir : localDirs) {
- Path userDir =
- new Path(localDir, ContainerLocalizer.USERCACHE);
- files.mkdir(userDir, new FsPermission("777"), false);
- // $local/filecache
- Path fileDir =
- new Path(localDir, ContainerLocalizer.FILECACHE);
- files.mkdir(fileDir, new FsPermission("777"), false);
- }
+
+ Assume.assumeTrue(shouldRun());
+
String locId = "container_01_01";
Path nmPrivateContainerTokensPath =
- dirsHandler.getLocalPathForWrite(
- ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
- + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
- locId));
+ dirsHandler
+ .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
+ + Path.SEPARATOR
+ + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId));
files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
Configuration config = new YarnConfiguration(conf);
- InetSocketAddress nmAddr = config.getSocketAddr(
- YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_LOCALIZER_ADDRESS,
- YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
- YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
+ InetSocketAddress nmAddr =
+ config.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_LOCALIZER_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
String appId = "application_01_01";
exec = new LinuxContainerExecutor() {
@Override
- public void buildMainArgs(List<String> command, String user, String appId,
- String locId, InetSocketAddress nmAddr, List<String> localDirs) {
- MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
- localDirs);
+ public void buildMainArgs(List<String> command, String user,
+ String appId, String locId, InetSocketAddress nmAddr,
+ List<String> localDirs) {
+ MockContainerLocalizer.buildMainArgs(command, user, appId, locId,
+ nmAddr, localDirs);
}
};
exec.setConf(conf);
@@ -277,44 +357,68 @@ public class TestLinuxContainerExecutor {
files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
appId, locId2, dirsHandler);
+ cleanupUserAppCache(appSubmitter);
}
-
+
@Test
- public void testContainerLaunch() throws IOException {
- if (!shouldRun()) {
- return;
- }
+ public void testContainerLaunch() throws Exception {
+ Assume.assumeTrue(shouldRun());
+ String expectedRunAsUser =
+ conf.get(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY,
+ YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
+
+ File touchFile = new File(workSpace, "touch-file");
+ int ret = runAndBlock("touch", touchFile.getAbsolutePath());
+
+ assertEquals(0, ret);
+ FileStatus fileStatus =
+ FileContext.getLocalFSFileContext().getFileStatus(
+ new Path(touchFile.getAbsolutePath()));
+ assertEquals(expectedRunAsUser, fileStatus.getOwner());
+ cleanupAppFiles(expectedRunAsUser);
+ }
+
+ @Test
+ public void testNonSecureRunAsSubmitter() throws Exception {
+ Assume.assumeTrue(shouldRun());
+ Assume.assumeFalse(UserGroupInformation.isSecurityEnabled());
+ String expectedRunAsUser = appSubmitter;
+ conf.set(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS, "false");
+ exec.setConf(conf);
File touchFile = new File(workSpace, "touch-file");
int ret = runAndBlock("touch", touchFile.getAbsolutePath());
-
+
assertEquals(0, ret);
- FileStatus fileStatus = FileContext.getLocalFSFileContext().getFileStatus(
+ FileStatus fileStatus =
+ FileContext.getLocalFSFileContext().getFileStatus(
new Path(touchFile.getAbsolutePath()));
- assertEquals(appSubmitter, fileStatus.getOwner());
+ assertEquals(expectedRunAsUser, fileStatus.getOwner());
+ cleanupAppFiles(expectedRunAsUser);
+ // reset conf
+ conf.unset(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS);
+ exec.setConf(conf);
}
@Test
public void testContainerKill() throws Exception {
- if (!shouldRun()) {
- return;
- }
-
- final ContainerId sleepId = getNextContainerId();
+ Assume.assumeTrue(shouldRun());
+
+ final ContainerId sleepId = getNextContainerId();
Thread t = new Thread() {
public void run() {
try {
runAndBlock(sleepId, "sleep", "100");
} catch (IOException e) {
- LOG.warn("Caught exception while running sleep",e);
+ LOG.warn("Caught exception while running sleep", e);
}
};
};
- t.setDaemon(true); //If it does not exit we shouldn't block the test.
+ t.setDaemon(true); // If it does not exit we shouldn't block the test.
t.start();
assertTrue(t.isAlive());
-
+
String pid = null;
int count = 10;
while ((pid = exec.getProcessId(sleepId)) == null && count > 0) {
@@ -328,40 +432,77 @@ public class TestLinuxContainerExecutor {
exec.signalContainer(appSubmitter, pid, Signal.TERM);
LOG.info("sleeping for 100ms to let the sleep be killed");
Thread.sleep(100);
-
+
assertFalse(t.isAlive());
+ cleanupAppFiles(appSubmitter);
+ }
+
+ @Test
+ public void testCGroups() throws Exception {
+ Assume.assumeTrue(shouldRun());
+ String cgroupsMount = System.getProperty("cgroups.mount");
+ Assume.assumeTrue((cgroupsMount != null) && !cgroupsMount.isEmpty());
+
+ assertTrue("Cgroups mount point does not exist", new File(
+ cgroupsMount).exists());
+ List<String> cgroupKVs = new ArrayList<>();
+
+ String hierarchy = "hadoop-yarn";
+ String[] controllers = { "cpu", "net_cls" };
+ for (String controller : controllers) {
+ cgroupKVs.add(controller + "=" + cgroupsMount + "/" + controller);
+ assertTrue(new File(cgroupsMount, controller).exists());
+ }
+
+ try {
+ exec.mountCgroups(cgroupKVs, hierarchy);
+ for (String controller : controllers) {
+ assertTrue(controller + " cgroup not mounted", new File(
+ cgroupsMount + "/" + controller + "/tasks").exists());
+ assertTrue(controller + " cgroup hierarchy not created",
+ new File(cgroupsMount + "/" + controller + "/" + hierarchy).exists());
+ assertTrue(controller + " cgroup hierarchy created incorrectly",
+ new File(cgroupsMount + "/" + controller + "/" + hierarchy
+ + "/tasks").exists());
+ }
+ } catch (IOException ie) {
+ fail("Couldn't mount cgroups " + ie.toString());
+ throw ie;
+ }
}
@Test
public void testLocalUser() throws Exception {
+ Assume.assumeTrue(shouldRun());
try {
- //nonsecure default
+ // nonsecure default
Configuration conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
+ "simple");
UserGroupInformation.setConfiguration(conf);
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.setConf(conf);
- Assert.assertEquals(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ Assert.assertEquals(
+ YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
lce.getRunAsUser("foo"));
- //nonsecure custom setting
+ // nonsecure custom setting
conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, "bar");
lce = new LinuxContainerExecutor();
lce.setConf(conf);
Assert.assertEquals("bar", lce.getRunAsUser("foo"));
- //nonsecure without limits
+ // nonsecure without limits
conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, "bar");
conf.setBoolean(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS, false);
lce = new LinuxContainerExecutor();
lce.setConf(conf);
Assert.assertEquals("foo", lce.getRunAsUser("foo"));
- //secure
+ // secure
conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "kerberos");
+ "kerberos");
UserGroupInformation.setConfiguration(conf);
lce = new LinuxContainerExecutor();
lce.setConf(conf);
@@ -369,49 +510,50 @@ public class TestLinuxContainerExecutor {
} finally {
Configuration conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
+ "simple");
UserGroupInformation.setConfiguration(conf);
}
}
@Test
public void testNonsecureUsernamePattern() throws Exception {
+ Assume.assumeTrue(shouldRun());
try {
- //nonsecure default
+ // nonsecure default
Configuration conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
+ "simple");
UserGroupInformation.setConfiguration(conf);
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.setConf(conf);
lce.verifyUsernamePattern("foo");
try {
lce.verifyUsernamePattern("foo/x");
- Assert.fail();
+ fail();
} catch (IllegalArgumentException ex) {
- //NOP
+ // NOP
} catch (Throwable ex) {
- Assert.fail(ex.toString());
+ fail(ex.toString());
}
-
- //nonsecure custom setting
+
+ // nonsecure custom setting
conf.set(YarnConfiguration.NM_NONSECURE_MODE_USER_PATTERN_KEY, "foo");
lce = new LinuxContainerExecutor();
lce.setConf(conf);
lce.verifyUsernamePattern("foo");
try {
lce.verifyUsernamePattern("bar");
- Assert.fail();
+ fail();
} catch (IllegalArgumentException ex) {
- //NOP
+ // NOP
} catch (Throwable ex) {
- Assert.fail(ex.toString());
+ fail(ex.toString());
}
- //secure, pattern matching does not kick in.
+ // secure, pattern matching does not kick in.
conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "kerberos");
+ "kerberos");
UserGroupInformation.setConfiguration(conf);
lce = new LinuxContainerExecutor();
lce.setConf(conf);
@@ -420,13 +562,14 @@ public class TestLinuxContainerExecutor {
} finally {
Configuration conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
+ "simple");
UserGroupInformation.setConfiguration(conf);
}
}
- @Test(timeout=10000)
+ @Test(timeout = 10000)
public void testPostExecuteAfterReacquisition() throws Exception {
+ Assume.assumeTrue(shouldRun());
// make up some bogus container ID
ApplicationId appId = ApplicationId.newInstance(12345, 67890);
ApplicationAttemptId attemptId =
@@ -435,7 +578,7 @@ public class TestLinuxContainerExecutor {
Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.NM_LINUX_CONTAINER_RESOURCES_HANDLER,
- TestResourceHandler.class, LCEResourcesHandler.class);
+ TestResourceHandler.class, LCEResourcesHandler.class);
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.setConf(conf);
try {
@@ -444,7 +587,7 @@ public class TestLinuxContainerExecutor {
// expected if LCE isn't setup right, but not necessary for this test
}
lce.reacquireContainer("foouser", cid);
- Assert.assertTrue("postExec not called after reacquisition",
+ assertTrue("postExec not called after reacquisition",
TestResourceHandler.postExecContainers.contains(cid));
}
[22/50] [abbrv] hadoop git commit: HDFS-8001 RpcProgramNfs3 : wrong
parsing of dfs.blocksize. Contributed by Remi Catherinot
Posted by zj...@apache.org.
HDFS-8001 RpcProgramNfs3 : wrong parsing of dfs.blocksize. Contributed by Remi Catherinot
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccf3fd1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccf3fd1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccf3fd1b
Branch: refs/heads/YARN-2928
Commit: ccf3fd1bcf8bb2f9c1c7ad6ef08a40e0b76aa4fe
Parents: 0968676
Author: Brandon Li <br...@apache.org>
Authored: Wed Apr 1 17:20:24 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccf3fd1b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
index 268abba..161f3e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
@@ -187,7 +187,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
clientCache = new DFSClientCache(config);
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
DFSConfigKeys.DFS_REPLICATION_DEFAULT);
- blockSize = config.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
+ blockSize = config.getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
bufferSize = config.getInt(
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccf3fd1b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 34c0556..518df9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1351,6 +1351,9 @@ Release 2.7.0 - UNRELEASED
HDFS-8036. Use snapshot path as source when using snapshot diff report in
DistCp. (Jing Zhao via wheat9)
+ HDFS-8001 RpcProgramNfs3 : wrong parsing of dfs.blocksize
+ (Remi Catherinot via brandonli)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
[45/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken
JDiff support in Hadoop 2. Contributed by Li Lu.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml
new file mode 100644
index 0000000..5514700
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.6.0.xml
@@ -0,0 +1,19520 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Mar 30 15:30:43 PDT 2015 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-hdfs 2.6.0"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/classes:/Users/llu/hadoop-common/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home/lib/tools.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-ker
beros-codec-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/llu/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/hadoop-common-2.6.0.jar:/Users/llu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/llu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/llu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/llu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/llu/.m2/repository/com/sun/jerse
y/jersey-json/1.9/jersey-json-1.9.jar:/Users/llu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/llu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/llu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/llu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/llu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/llu/.m2/repository/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar:/Users/llu/.m2/repository/commons-el/commons-el/1.0/commons-el-1.0.jar:/Users/llu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/llu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/llu/.m2/repository/commons-configuration/commons-configuratio
n/1.6/commons-configuration-1.6.jar:/Users/llu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/llu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/llu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/llu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/llu/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/llu/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/llu/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/llu/.m2/repository/org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/llu/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/llu/.m2/repository/
org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/llu/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/llu/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/Users/llu/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/llu/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/llu/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/llu/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/llu/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/llu/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/llu/.m2/repository/commons-logging/commons-logging/1.1.3/commo
ns-logging-1.1.3.jar:/Users/llu/.m2/repository/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/Users/llu/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/llu/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/llu/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/llu/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/llu/.m2/repository/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar:/Users/llu/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/llu/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/llu/.m2/repository/xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar:/Users/llu/.m2/re
pository/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar:/Users/llu/.m2/repository/org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar -sourcepath /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/src/main/java -apidir /Users/llu/hadoop-common/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname hadoop-core 2.6.0 -->
+<package name="org.apache.hadoop.fs">
+ <!-- start class org.apache.hadoop.fs.BlockStorageLocation -->
+ <class name="BlockStorageLocation" extends="org.apache.hadoop.fs.BlockLocation"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="BlockStorageLocation" type="org.apache.hadoop.fs.BlockLocation, org.apache.hadoop.fs.VolumeId[]"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getVolumeIds" return="org.apache.hadoop.fs.VolumeId[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the list of {@link VolumeId} corresponding to the block's replicas.
+
+ @return volumeIds list of VolumeId for the block's replicas]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Wrapper for {@link BlockLocation} that also adds {@link VolumeId} volume
+ location information for each replica.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.BlockStorageLocation -->
+ <!-- start class org.apache.hadoop.fs.CacheFlag -->
+ <class name="CacheFlag" extends="java.lang.Enum"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.fs.CacheFlag[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.fs.CacheFlag"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <doc>
+ <![CDATA[Specifies semantics for CacheDirective operations. Multiple flags can
+ be combined in an EnumSet.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.CacheFlag -->
+ <!-- start class org.apache.hadoop.fs.HdfsVolumeId -->
+ <class name="HdfsVolumeId" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.VolumeId"/>
+ <constructor name="HdfsVolumeId" type="byte[]"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="compareTo" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.fs.VolumeId"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="obj" type="java.lang.Object"/>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[HDFS-specific volume identifier which implements {@link VolumeId}. Can be
+ used to differentiate between the data directories on a single datanode. This
+ identifier is only unique on a per-datanode basis.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.HdfsVolumeId -->
+ <!-- start interface org.apache.hadoop.fs.VolumeId -->
+ <interface name="VolumeId" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Comparable"/>
+ <method name="compareTo" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.fs.VolumeId"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="obj" type="java.lang.Object"/>
+ </method>
+ <doc>
+ <![CDATA[Opaque interface that identifies a disk location. Subclasses
+ should implement {@link Comparable} and override both equals and hashCode.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.fs.VolumeId -->
+ <!-- start class org.apache.hadoop.fs.XAttr.Builder -->
+ <class name="XAttr.Builder" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="XAttr.Builder"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setNameSpace" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ns" type="org.apache.hadoop.fs.XAttr.NameSpace"/>
+ </method>
+ <method name="setName" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <method name="setValue" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="byte[]"/>
+ </method>
+ <method name="build" return="org.apache.hadoop.fs.XAttr"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.fs.XAttr.Builder -->
+ <!-- start class org.apache.hadoop.fs.XAttr.NameSpace -->
+ <class name="XAttr.NameSpace" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.fs.XAttr.NameSpace[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.fs.XAttr.NameSpace"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.fs.XAttr.NameSpace -->
+</package>
+<package name="org.apache.hadoop.hdfs">
+ <!-- start interface org.apache.hadoop.hdfs.BlockReader -->
+ <interface name="BlockReader" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+ <method name="read" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="off" type="int"/>
+ <param name="len" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="skip" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="n" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Skip the given number of bytes]]>
+ </doc>
+ </method>
+ <method name="available" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns an estimate of the number of bytes that can be read
+ (or skipped over) from this input stream without performing
+ network I/O.
+ This may return more than what is actually present in the block.]]>
+ </doc>
+ </method>
+ <method name="close"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Close the block reader.
+
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="readFully"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="readOffset" type="int"/>
+ <param name="amtToRead" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Read exactly the given amount of data, throwing an exception
+ if EOF is reached before that amount]]>
+ </doc>
+ </method>
+ <method name="readAll" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="offset" type="int"/>
+ <param name="len" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Similar to {@link #readFully(byte[], int, int)} except that it will
+ not throw an exception on EOF. However, it differs from the simple
+ {@link #read(byte[], int, int)} call in that it is guaranteed to
+ read the data if it is available. In other words, if this call
+ does not throw an exception, then either the buffer has been
+ filled or the next call will return EOF.]]>
+ </doc>
+ </method>
+ <method name="isLocal" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true only if this is a local read.]]>
+ </doc>
+ </method>
+ <method name="isShortCircuit" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true only if this is a short-circuit read.
+ All short-circuit reads are also local.]]>
+ </doc>
+ </method>
+ <method name="getClientMmap" return="org.apache.hadoop.hdfs.shortcircuit.ClientMmap"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="opts" type="java.util.EnumSet"/>
+ <doc>
+ <![CDATA[Get a ClientMmap object for this BlockReader.
+
+ @param opts The read options to use.
+ @return The ClientMmap object, or null if mmap is not
+ supported.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[A BlockReader is responsible for reading a single block
+ from a single datanode.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.BlockReader -->
+ <!-- start class org.apache.hadoop.hdfs.BlockReaderFactory.BlockReaderPeer -->
+ <class name="BlockReaderFactory.BlockReaderPeer" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.BlockReaderFactory.BlockReaderPeer -->
+ <!-- start class org.apache.hadoop.hdfs.CorruptFileBlockIterator -->
+ <class name="CorruptFileBlockIterator" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.RemoteIterator"/>
+ <constructor name="CorruptFileBlockIterator" type="org.apache.hadoop.hdfs.DFSClient, org.apache.hadoop.fs.Path"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getCallsMade" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the number of calls made to the DFSClient.
+ This is for debugging and testing purposes.]]>
+ </doc>
+ </method>
+ <method name="hasNext" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="next" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <doc>
+ <![CDATA[Provides an iterator interface for listCorruptFileBlocks.
+ This class is used by DistributedFileSystem and Hdfs.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.CorruptFileBlockIterator -->
+ <!-- start class org.apache.hadoop.hdfs.DFSClient.Conf -->
+ <class name="DFSClient.Conf" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DFSClient.Conf" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="isUseLegacyBlockReaderLocal" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDomainSocketPath" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isShortCircuitLocalReads" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isDomainSocketDataTraffic" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[DFSClient configuration]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSClient.Conf -->
+ <!-- start class org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream -->
+ <class name="DFSClient.DFSDataInputStream" extends="org.apache.hadoop.hdfs.client.HdfsDataInputStream"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link HdfsDataInputStream} instead.">
+ <constructor name="DFSClient.DFSDataInputStream" type="org.apache.hadoop.hdfs.DFSInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <doc>
+ <![CDATA[@deprecated use {@link HdfsDataInputStream} instead.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.DFSHedgedReadMetrics -->
+ <class name="DFSHedgedReadMetrics" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DFSHedgedReadMetrics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="incHedgedReadOps"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incHedgedReadOpsInCurThread"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incHedgedReadWins"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadOps" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadOpsInCurThread" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadWins" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <field name="hedgedReadOps" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="hedgedReadOpsWin" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="hedgedReadOpsInCurThread" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[The client-side metrics for hedged read feature.
+ This class has a number of metrics variables that are publicly accessible,
+ we can grab them from client side, like HBase.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSHedgedReadMetrics -->
+ <!-- start class org.apache.hadoop.hdfs.DFSInotifyEventInputStream -->
+ <class name="DFSInotifyEventInputStream" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="poll" return="org.apache.hadoop.hdfs.inotify.Event"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next event in the stream or null if no new events are currently
+ available.
+
+ @throws IOException because of network error or edit log
+ corruption. Also possible if JournalNodes are unresponsive in the
+ QJM setting (even one unresponsive JournalNode is enough in rare cases),
+ so catching this exception and retrying at least a few times is
+ recommended.
+ @throws MissingEventsException if we cannot return the next event in the
+ stream because the data for the event (and possibly some subsequent events)
+ has been deleted (generally because this stream is a very large number of
+ events behind the current state of the NameNode). It is safe to continue
+ reading from the stream after this exception is thrown -- the next
+ available event will be returned.]]>
+ </doc>
+ </method>
+ <method name="getEventsBehindEstimate" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return a estimate of how many events behind the NameNode's current state
+ this stream is. Clients should periodically call this method and check if
+ its result is steadily increasing, which indicates that they are falling
+ behind (i.e. events are being generated faster than the client is reading
+ them). If a client falls too far behind events may be deleted before the
+ client can read them.
+ <p/>
+ A return value of -1 indicates that an estimate could not be produced, and
+ should be ignored. The value returned by this method is really only useful
+ when compared to previous or subsequent returned values.]]>
+ </doc>
+ </method>
+ <method name="poll" return="org.apache.hadoop.hdfs.inotify.Event"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="time" type="long"/>
+ <param name="tu" type="java.util.concurrent.TimeUnit"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next event in the stream, waiting up to the specified amount of
+ time for a new event. Returns null if a new event is not available at the
+ end of the specified amount of time. The time before the method returns may
+ exceed the specified amount of time by up to the time required for an RPC
+ to the NameNode.
+
+ @param time number of units of the given TimeUnit to wait
+ @param tu the desired TimeUnit
+ @throws IOException see {@link DFSInotifyEventInputStream#poll()}
+ @throws MissingEventsException
+ see {@link DFSInotifyEventInputStream#poll()}
+ @throws InterruptedException if the calling thread is interrupted]]>
+ </doc>
+ </method>
+ <method name="take" return="org.apache.hadoop.hdfs.inotify.Event"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next event in the stream, waiting indefinitely if a new event
+ is not immediately available.
+
+ @throws IOException see {@link DFSInotifyEventInputStream#poll()}
+ @throws MissingEventsException see
+ {@link DFSInotifyEventInputStream#poll()}
+ @throws InterruptedException if the calling thread is interrupted]]>
+ </doc>
+ </method>
+ <field name="LOG" type="org.slf4j.Logger"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[Stream for reading inotify events. DFSInotifyEventInputStreams should not
+ be shared among multiple threads.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSInotifyEventInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics -->
+ <class name="DFSInputStream.ReadStatistics" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DFSInputStream.ReadStatistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="DFSInputStream.ReadStatistics" type="org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getTotalBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total bytes read. This will always be at least as
+ high as the other numbers, since it includes all of them.]]>
+ </doc>
+ </method>
+ <method name="getTotalLocalBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total local bytes read. This will always be at least
+ as high as totalShortCircuitBytesRead, since all short-circuit
+ reads are also local.]]>
+ </doc>
+ </method>
+ <method name="getTotalShortCircuitBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total short-circuit local bytes read.]]>
+ </doc>
+ </method>
+ <method name="getTotalZeroCopyBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total number of zero-copy bytes read.]]>
+ </doc>
+ </method>
+ <method name="getRemoteBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total number of bytes read which were not local.]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics -->
+ <!-- start class org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress -->
+ <class name="DFSUtil.ConfiguredNNAddress" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getNameserviceId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getNamenodeId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Represent one of the NameNodes configured in the cluster.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress -->
+ <!-- start class org.apache.hadoop.hdfs.ExtendedBlockId -->
+ <class name="ExtendedBlockId" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ExtendedBlockId" type="long, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="fromExtendedBlock" return="org.apache.hadoop.hdfs.ExtendedBlockId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="block" type="org.apache.hadoop.hdfs.protocol.ExtendedBlock"/>
+ </method>
+ <method name="getBlockId" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getBlockPoolId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="o" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[An immutable key which identifies a block.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.ExtendedBlockId -->
+ <!-- start class org.apache.hadoop.hdfs.HAUtil -->
+ <class name="HAUtil" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="isHAEnabled" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Returns true if HA for namenode is configured for the given nameservice
+
+ @param conf Configuration
+ @param nsId nameservice, or null if no federated NS is configured
+ @return true if HA is configured in the configuration; else false.]]>
+ </doc>
+ </method>
+ <method name="usesSharedEditsDir" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Returns true if HA is using a shared edits directory.
+
+ @param conf Configuration
+ @return true if HA config is using a shared edits dir, false otherwise.]]>
+ </doc>
+ </method>
+ <method name="getNameNodeId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the namenode Id by matching the {@code addressKey}
+ with the the address of the local node.
+
+ If {@link DFSConfigKeys#DFS_HA_NAMENODE_ID_KEY} is not specifically
+ configured, this method determines the namenode Id by matching the local
+ node's address with the configured addresses. When a match is found, it
+ returns the namenode Id from the corresponding configuration key.
+
+ @param conf Configuration
+ @return namenode Id on success, null on failure.
+ @throws HadoopIllegalArgumentException on error]]>
+ </doc>
+ </method>
+ <method name="getNameNodeIdFromAddress" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="address" type="java.net.InetSocketAddress"/>
+ <param name="keys" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Similar to
+ {@link DFSUtil#getNameServiceIdFromAddress(Configuration,
+ InetSocketAddress, String...)}]]>
+ </doc>
+ </method>
+ <method name="getNameNodeIdOfOtherNode" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the NN ID of the other node in an HA setup.
+
+ @param conf the configuration of this node
+ @return the NN ID of the other node in this nameservice]]>
+ </doc>
+ </method>
+ <method name="getConfForOtherNode" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="myConf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Given the configuration for this node, return a Configuration object for
+ the other node in an HA setup.
+
+ @param myConf the configuration of this node
+ @return the configuration of the other node in an HA setup]]>
+ </doc>
+ </method>
+ <method name="shouldAllowStandbyReads" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[This is used only by tests at the moment.
+ @return true if the NN should allow read operations while in standby mode.]]>
+ </doc>
+ </method>
+ <method name="setAllowStandbyReads"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="val" type="boolean"/>
+ </method>
+ <method name="isLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <doc>
+ <![CDATA[@return true if the given nameNodeUri appears to be a logical URI.]]>
+ </doc>
+ </method>
+ <method name="isClientFailoverConfigured" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <doc>
+ <![CDATA[Check whether the client has a failover proxy provider configured
+ for the namenode/nameservice.
+
+ @param conf Configuration
+ @param nameNodeUri The URI of namenode
+ @return true if failover is configured.]]>
+ </doc>
+ </method>
+ <method name="useLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Check whether logical URI is needed for the namenode and
+ the corresponding failover proxy provider in the config.
+
+ @param conf Configuration
+ @param nameNodeUri The URI of namenode
+ @return true if logical URI is needed. false, if not needed.
+ @throws IOException most likely due to misconfiguration.]]>
+ </doc>
+ </method>
+ <method name="getServiceUriFromToken" return="java.net.URI"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="scheme" type="java.lang.String"/>
+ <param name="token" type="org.apache.hadoop.security.token.Token"/>
+ <doc>
+ <![CDATA[Parse the file system URI out of the provided token.]]>
+ </doc>
+ </method>
+ <method name="buildTokenServiceForLogicalUri" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="uri" type="java.net.URI"/>
+ <param name="scheme" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the service name used in the delegation token for the given logical
+ HA service.
+ @param uri the logical URI of the cluster
+ @param scheme the scheme of the corresponding FileSystem
+ @return the service name]]>
+ </doc>
+ </method>
+ <method name="isTokenForLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="token" type="org.apache.hadoop.security.token.Token"/>
+ <doc>
+ <![CDATA[@return true if this token corresponds to a logical nameservice
+ rather than a specific namenode.]]>
+ </doc>
+ </method>
+ <method name="buildTokenServicePrefixForLogicalUri" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="scheme" type="java.lang.String"/>
+ </method>
+ <method name="cloneDelegationTokenForLogicalUri"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="haUri" type="java.net.URI"/>
+ <param name="nnAddrs" type="java.util.Collection"/>
+ <doc>
+ <![CDATA[Locate a delegation token associated with the given HA cluster URI, and if
+ one is found, clone it to also represent the underlying namenode address.
+ @param ugi the UGI to modify
+ @param haUri the logical URI for the cluster
+ @param nnAddrs collection of NNs in the cluster to which the token
+ applies]]>
+ </doc>
+ </method>
+ <method name="getAddressOfActive" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the internet address of the currently-active NN. This should rarely be
+ used, since callers of this method who connect directly to the NN using the
+ resulting InetSocketAddress will not be able to connect to the active NN if
+ a failover were to occur after this method has been called.
+
+ @param fs the file system to get the active address of.
+ @return the internet address of the currently-active NN.
+ @throws IOException if an error occurs while resolving the active NN.]]>
+ </doc>
+ </method>
+ <method name="getProxiesForAllNameNodesInNameservice" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ call should be made on every NN in an HA nameservice, not just the active.
+
+ @param conf configuration
+ @param nsId the nameservice to get all of the proxies for.
+ @return a list of RPC proxies for each NN in the nameservice.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ <method name="getProxiesForAllNameNodesInNameservice" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <param name="xface" type="java.lang.Class"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ call should be made on every NN in an HA nameservice, not just the active.
+
+ @param conf configuration
+ @param nsId the nameservice to get all of the proxies for.
+ @param xface the protocol class.
+ @return a list of RPC proxies for each NN in the nameservice.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ <method name="isAtLeastOneActive" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="namenodes" type="java.util.List"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Used to ensure that at least one of the given HA NNs is currently in the
+ active state..
+
+ @param namenodes list of RPC proxies for each NN to check.
+ @return true if at least one NN is active, false if all are in the standby state.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.HAUtil -->
+ <!-- start class org.apache.hadoop.hdfs.NameNodeProxies -->
+ <class name="NameNodeProxies" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="NameNodeProxies"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="createProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the namenode proxy with the passed protocol. This will handle
+ creation of either HA- or non-HA-enabled proxy objects, depending upon
+ if the provided URI is a configured logical URI.
+
+ @param conf the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the namenode proxy with the passed protocol. This will handle
+ creation of either HA- or non-HA-enabled proxy objects, depending upon
+ if the provided URI is a configured logical URI.
+
+ @param conf the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createProxyWithLossyRetryHandler" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="numResponseToDrop" type="int"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Generate a dummy namenode proxy instance that utilizes our hacked
+ {@link LossyRetryInvocationHandler}. Proxy instance generated using this
+ method will proactively drop RPC responses. Currently this method only
+ support HA setup. null will be returned if the given configuration is not
+ for HA.
+
+ @param config the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @param numResponseToDrop The number of responses to drop for each RPC call
+ @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to. Will return null of the
+ given configuration does not support HA.
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createNonHAProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nnAddr" type="java.net.InetSocketAddress"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="withRetries" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+
+ @param conf the configuration object
+ @param nnAddr address of the remote NN to connect to
+ @param xface the IPC interface which should be created
+ @param ugi the user who is making the calls on the proxy object
+ @param withRetries certain interfaces have a non-standard retry policy
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="createNonHAProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nnAddr" type="java.net.InetSocketAddress"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="withRetries" type="boolean"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+
+ @param conf the configuration object
+ @param nnAddr address of the remote NN to connect to
+ @param xface the IPC interface which should be created
+ @param ugi the user who is making the calls on the proxy object
+ @param withRetries certain interfaces have a non-standard retry policy
+ @param fallbackToSimpleAuth - set to true or false during this method to
+ indicate if a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getFailoverProxyProviderClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Gets the configured Failover proxy provider's class]]>
+ </doc>
+ </method>
+ <method name="createFailoverProxyProvider" return="org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="checkPort" type="boolean"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the Failover proxy provider instance]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Create proxy objects to communicate with a remote NN. All remote access to an
+ NN should be funneled through this class. Most of the time you'll want to use
+ {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
+ create either an HA- or non-HA-enabled client proxy as appropriate.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.NameNodeProxies -->
+ <!-- start class org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo -->
+ <class name="NameNodeProxies.ProxyAndInfo" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="NameNodeProxies.ProxyAndInfo" type="PROXYTYPE, org.apache.hadoop.io.Text, java.net.InetSocketAddress"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getProxy" return="PROXYTYPE"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDelegationTokenService" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Wrapper for a client proxy as well as its associated service ID.
+ This is simply used as a tuple-like return type for
+ {@link NameNodeProxies#createProxy} and
+ {@link NameNodeProxies#createNonHAProxy}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo -->
+ <!-- start interface org.apache.hadoop.hdfs.RemotePeerFactory -->
+ <interface name="RemotePeerFactory" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="newConnectedPeer" return="org.apache.hadoop.hdfs.net.Peer"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <param name="blockToken" type="org.apache.hadoop.security.token.Token"/>
+ <param name="datanodeId" type="org.apache.hadoop.hdfs.protocol.DatanodeID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[@param addr The address to connect to.
+ @param blockToken Token used during optional SASL negotiation
+ @param datanodeId ID of destination DataNode
+ @return A new Peer connected to the address.
+
+ @throws IOException If there was an error connecting or creating
+ the remote socket, encrypted stream, etc.]]>
+ </doc>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.RemotePeerFactory -->
+ <!-- start class org.apache.hadoop.hdfs.StorageType -->
+ <class name="StorageType" extends="java.lang.Enum"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.hdfs.StorageType[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.hdfs.StorageType"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <method name="isTransient" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isMovable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="asList" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getMovableTypes" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <field name="DEFAULT" type="org.apache.hadoop.hdfs.StorageType"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="EMPTY_ARRAY" type="org.apache.hadoop.hdfs.StorageType[]"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[Defines the types of supported storage media. The default storage
+ medium is assumed to be DISK.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.StorageType -->
+ <!-- start class org.apache.hadoop.hdfs.UnknownCipherSuiteException -->
+ <class name="UnknownCipherSuiteException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="UnknownCipherSuiteException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Thrown when an unknown cipher suite is encountered.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.UnknownCipherSuiteException -->
+ <!-- start class org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException -->
+ <class name="UnknownCryptoProtocolVersionException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="UnknownCryptoProtocolVersionException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="UnknownCryptoProtocolVersionException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException -->
+ <doc>
+ <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}. This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files
+have strictly one writer at any one time. Bytes are always appended
+to the end of the writer's stream. There is no notion of "record appends"
+or "mutations" that are then checked or reordered. Writers simply emit
+a byte stream. That byte stream is guaranteed to be stored in the
+order written.</p>]]>
+ </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.client">
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsAdmin -->
+ <class name="HdfsAdmin" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsAdmin" type="java.net.URI, org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Create a new HdfsAdmin client.
+
+ @param uri the unique URI of the HDFS file system to administer
+ @param conf configuration
+ @throws IOException in the event the file system could not be created]]>
+ </doc>
+ </constructor>
+ <method name="setQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="quota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the namespace quota (count of files, directories, and sym links) for a
+ directory.
+
+ @param src the path to set the quota for
+ @param quota the value to set for the quota
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the namespace quota (count of files, directories and sym links) for a
+ directory.
+
+ @param src the path to clear the quota of
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="setSpaceQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="spaceQuota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the disk space quota (size of files) for a directory. Note that
+ directories and sym links do not occupy disk space.
+
+ @param src the path to set the space quota of
+ @param spaceQuota the value to set for the space quota
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearSpaceQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the disk space quota (size of files) for a directory. Note that
+ directories and sym links do not occupy disk space.
+
+ @param src the path to clear the space quota of
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="allowSnapshot"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Allow snapshot on a directory.
+ @param path The path of the directory where snapshots will be taken.]]>
+ </doc>
+ </method>
+ <method name="disallowSnapshot"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Disallow snapshot on a directory.
+ @param path The path of the snapshottable directory.]]>
+ </doc>
+ </method>
+ <method name="addCacheDirective" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <param name="flags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add a new CacheDirectiveInfo.
+
+ @param info Information about a directive to add.
+ @param flags {@link CacheFlag}s to use for this operation.
+ @return the ID of the directive that was created.
+ @throws IOException if the directive could not be added]]>
+ </doc>
+ </method>
+ <method name="modifyCacheDirective"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <param name="flags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Modify a CacheDirective.
+
+ @param info Information about the directive to modify. You must set the ID
+ to indicate which CacheDirective you want to modify.
+ @param flags {@link CacheFlag}s to use for this operation.
+ @throws IOException if the directive could not be modified]]>
+ </doc>
+ </method>
+ <method name="removeCacheDirective"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Remove a CacheDirective.
+
+ @param id identifier of the CacheDirectiveInfo to remove
+ @throws IOException if the directive could not be removed]]>
+ </doc>
+ </method>
+ <method name="listCacheDirectives" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="filter" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List cache directives. Incrementally fetches results from the server.
+
+ @param filter Filter parameters to use when listing the directives, null to
+ list all directives visible to us.
+ @return A RemoteIterator which returns CacheDirectiveInfo objects.]]>
+ </doc>
+ </method>
+ <method name="addCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CachePoolInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add a cache pool.
+
+ @param info
+ The request to add a cache pool.
+ @throws IOException
+ If the request could not be completed.]]>
+ </doc>
+ </method>
+ <method name="modifyCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CachePoolInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Modify an existing cache pool.
+
+ @param info
+ The request to modify a cache pool.
+ @throws IOException
+ If the request could not be completed.]]>
+ </doc>
+ </method>
+ <method name="removeCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="poolName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Remove a cache pool.
+
+ @param poolName
+ Name of the cache pool to remove.
+ @throws IOException
+ if the cache pool did not exist, or could not be removed.]]>
+ </doc>
+ </method>
+ <method name="listCachePools" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List all cache pools.
+
+ @return A remote iterator from which you can get CachePoolEntry objects.
+ Requests will be made as needed.
+ @throws IOException
+ If there was an error listing cache pools.]]>
+ </doc>
+ </method>
+ <method name="createEncryptionZone"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <param name="keyName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+ <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+ <doc>
+ <![CDATA[Create an encryption zone rooted at an empty existing directory, using the
+ specified encryption key. An encryption zone has an associated encryption
+ key used when reading and writing files within the zone.
+
+ @param path The path of the root of the encryption zone. Must refer to
+ an empty, existing directory.
+ @param keyName Name of key available at the KeyProvider.
+ @throws IOException if there was a general IO exception
+ @throws AccessControlException if the caller does not have access to path
+ @throws FileNotFoundException if the path does not exist]]>
+ </doc>
+ </method>
+ <method name="getEncryptionZoneForPath" return="org.apache.hadoop.hdfs.protocol.EncryptionZone"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+ <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+ <doc>
+ <![CDATA[Get the path of the encryption zone for a given file or directory.
+
+ @param path The path to get the ez for.
+
+ @return The EncryptionZone of the ez, or null if path is not in an ez.
+ @throws IOException if there was a general IO exception
+ @throws AccessControlException if the caller does not have access to path
+ @throws FileNotFoundException if the path does not exist]]>
+ </doc>
+ </method>
+ <method name="listEncryptionZones" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns a RemoteIterator which can be used to list the encryption zones
+ in HDFS. For large numbers of encryption zones, the iterator will fetch
+ the list of zones in a number of small batches.
+ <p/>
+ Since the list is fetched in batches, it does not represent a
+ consistent snapshot of the entire list of encryption zones.
+ <p/>
+ This method can only be called by HDFS superusers.]]>
+ </doc>
+ </method>
+ <method name="getInotifyEventStream" return="org.apache.hadoop.hdfs.DFSInotifyEventInputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Exposes a stream of namesystem events. Only events occurring after the
+ stream is created are available.
+ See {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream}
+ for information on stream usage.
+ See {@link org.apache.hadoop.hdfs.inotify.Event}
+ for information on the available events.
+ <p/>
+ Inotify users may want to tune the following HDFS parameters to
+ ensure that enough extra HDFS edits are saved to support inotify clients
+ that fall behind the current state of the namespace while reading events.
+ The default parameter values should generally be reasonable. If edits are
+ deleted before their corresponding events can be read, clients will see a
+ {@link org.apache.hadoop.hdfs.inotify.MissingEventsException} on
+ {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream} method calls.
+
+ It should generally be sufficient to tune these parameters:
+ dfs.namenode.num.extra.edits.retained
+ dfs.namenode.max.extra.edits.segments.retained
+
+ Parameters that affect the number of created segments and the number of
+ edits that are considered necessary, i.e. do not count towards the
+ dfs.namenode.num.extra.edits.retained quota):
+ dfs.namenode.checkpoint.period
+ dfs.namenode.checkpoint.txns
+ dfs.namenode.num.checkpoints.retained
+ dfs.ha.log-roll.period
+ <p/>
+ It is recommended that local journaling be configured
+ (dfs.namenode.edits.dir) for inotify (in addition to a shared journal)
+ so that edit transfers from the shared journal can be avoided.
+
+ @throws IOException If there was an error obtaining the stream.]]>
+ </doc>
+ </method>
+ <method name="getInotifyEventStream" return="org.apache.hadoop.hdfs.DFSInotifyEventInputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="lastReadTxid" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[A version of {@link HdfsAdmin#getInotifyEventStream()} meant for advanced
+ users who are aware of HDFS edits up to lastReadTxid (e.g. because they
+ have access to an FSImage inclusive of lastReadTxid) and only want to read
+ events after this point.]]>
+ </doc>
+ </method>
+ <method name="setStoragePolicy"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="policyName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the source path to the specified storage policy.
+
+ @param src The source path referring to either a directory or a file.
+ @param policyName The name of the storage policy.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[The public API for performing administrative functions on HDFS. Those writing
+ applications against HDFS should prefer this interface to directly accessing
+ functionality in DistributedFileSystem or DFSClient.
+
+ Note that this is distinct from the similarly-named {@link DFSAdmin}, which
+ is a class that provides the functionality for the CLI `hdfs dfsadmin ...'
+ commands.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.HdfsAdmin -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsDataInputStream -->
+ <class name="HdfsDataInputStream" extends="org.apache.hadoop.fs.FSDataInputStream"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsDataInputStream" type="org.apache.hadoop.hdfs.DFSInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <constructor name="HdfsDataInputStream" type="org.apache.hadoop.crypto.CryptoInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getWrappedStream" return="java.io.InputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get a reference to the wrapped output stream. We always want to return the
+ actual underlying InputStream, even when we're using a CryptoStream. e.g.
+ in the delegated methods below.
+
+ @return the underlying output stream]]>
+ </doc>
+ </method>
+ <method name="getCurrentDatanode" return="org.apache.hadoop.hdfs.protocol.DatanodeInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the datanode from which the stream is currently reading.]]>
+ </doc>
+ </method>
+ <method name="getCurrentBlock" return="org.apache.hadoop.hdfs.protocol.ExtendedBlock"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the block containing the target position.]]>
+ </doc>
+ </method>
+ <method name="getAllBlocks" return="java.util.List"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the collection of blocks that has already been located.]]>
+ </doc>
+ </method>
+ <method name="getVisibleLength" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the visible length of the file. It will include the length of the last
+ block even if that is in UnderConstruction state.
+
+ @return The visible length of the file.]]>
+ </doc>
+ </method>
+ <method name="getReadStatistics" return="org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get statistics about the reads which this DFSInputStream has done.
+ Note that because HdfsDataInputStream is buffered, these stats may
+ be higher than you would expect just by adding up the number of
+ bytes read through HdfsDataInputStream.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[The Hdfs implementation of {@link FSDataInputStream}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.HdfsDataInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsDataOutputStream -->
+ <class name="HdfsDataOutputStream" extends="org.apache.hadoop.fs.FSDataOutputStream"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsDataOutputStream" type="org.apache.hadoop.hdfs.DFSOutputStream, org.apache.hadoop.fs.FileSystem.Statistics, long"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <constructor name="HdfsDataOutputStream" type="org.apache.hadoop.hdfs.DFSOutputStream, org.apache.hadoop.fs.FileSystem.Statistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <constructor name="HdfsDataOutputStream" type="org.apache.hadoop.crypto.CryptoOutputStream, org.apache.hadoop.fs.FileSystem.Statistics, long"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <constructor name="HdfsDataOutputStream" type="org.apache.hadoop.crypto.CryptoOutputStream, org.apache.hadoop.fs.FileSystem.Statistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getCurrentBlockReplication" return="int"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the actual number of replicas of the current block.
+
+ This can be different from the designated replication factor of the file
+ because the namenode does not maintain replication for the blocks which are
+ currently being written to. Depending on the configuration, the client may
+ continue to write to a block even if a few datanodes in the write pipeline
+ have failed, or the client may add a new datanodes once a datanode has
+ failed.
+
+ @return the number of valid replicas of the current block]]>
+ </doc>
+ </method>
+ <method name="hsync"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="syncFlags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Sync buffered data to DataNodes (flush to disk devices).
+
+ @param syncFlags
+ Indicate the detailed semantic and actions of the hsync.
+ @throws IOException
+ @see FSDataOutputStream#hsync()]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[The Hdfs implementation of {@link FSDataOutputStream}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.HdfsDataOutputStream -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag -->
+ <class name="HdfsDataOutputStream.SyncFlag" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag[]"
+ abstract="false" native="false" synchronized="false"
+ static="t
<TRUNCATED>
[47/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken
JDiff support in Hadoop 2. Contributed by Li Lu.
Posted by zj...@apache.org.
HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1069dcee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1069dcee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1069dcee
Branch: refs/heads/YARN-2928
Commit: 1069dceea24fe019e77619a545909e52430b4275
Parents: 3485a8d
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Sat Apr 4 13:52:01 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:17 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../jdiff/Apache_Hadoop_Common_2.6.0.xml | 45596 +++++++++++++++++
.../jdiff/Apache_Hadoop_HDFS_2.6.0.xml | 19520 +++++++
hadoop-project-dist/pom.xml | 24 +-
hadoop-project/pom.xml | 2 +
5 files changed, 65137 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index fd926aa..f52e09f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1205,6 +1205,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11377. Added Null.java without which jdiff completely flops. (Tsuyoshi
Ozawa via vinodkv)
+
+ HADOOP-11776. Fixed the broken JDiff support in Hadoop 2. (Li Lu via vinodkv)
Release 2.6.1 - UNRELEASED
[17/50] [abbrv] hadoop git commit: HDFS-8008. Support client-side
back off when the datanodes are congested. Contributed by Haohui Mai.
Posted by zj...@apache.org.
HDFS-8008. Support client-side back off when the datanodes are congested. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6d4143f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6d4143f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6d4143f
Branch: refs/heads/YARN-2928
Commit: f6d4143f32845a4230919c998269f59b9f3be11b
Parents: 61487d3
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Apr 1 16:54:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DataStreamer.java | 63 ++++++++++++++++++++
.../hdfs/protocol/datatransfer/PipelineAck.java | 4 ++
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 42 +++++++++++++
4 files changed, 112 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d4143f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1d9e200..34c0556 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -868,6 +868,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7742. Favoring decommissioning node for replication can cause a block
to stay underreplicated for long periods (Nathan Roberts via kihwal)
+ HDFS-8008. Support client-side back off when the datanodes are congested.
+ (wheat9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d4143f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 9c437ba..6ff4c24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -218,6 +218,13 @@ class DataStreamer extends Daemon {
private boolean failPacket = false;
private final long dfsclientSlowLogThresholdMs;
private long artificialSlowdown = 0;
+ // List of congested data nodes. The stream will back off if the DataNodes
+ // are congested
+ private final ArrayList<DatanodeInfo> congestedNodes = new ArrayList<>();
+ private static final int CONGESTION_BACKOFF_MEAN_TIME_IN_MS = 5000;
+ private static final int CONGESTION_BACK_OFF_MAX_TIME_IN_MS =
+ CONGESTION_BACKOFF_MEAN_TIME_IN_MS * 10;
+ private int lastCongestionBackoffTime;
private final LoadingCache<DatanodeInfo, DatanodeInfo> excludedNodes;
@@ -386,6 +393,11 @@ class DataStreamer extends Daemon {
one = createHeartbeatPacket();
assert one != null;
} else {
+ try {
+ backOffIfNecessary();
+ } catch (InterruptedException e) {
+ DFSClient.LOG.warn("Caught exception ", e);
+ }
one = dataQueue.getFirst(); // regular data packet
long parents[] = one.getTraceParents();
if (parents.length > 0) {
@@ -815,9 +827,14 @@ class DataStreamer extends Daemon {
long seqno = ack.getSeqno();
// processes response status from datanodes.
+ ArrayList<DatanodeInfo> congestedNodesFromAck = new ArrayList<>();
for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) {
final Status reply = PipelineAck.getStatusFromHeader(ack
.getHeaderFlag(i));
+ if (PipelineAck.getECNFromHeader(ack.getHeaderFlag(i)) ==
+ PipelineAck.ECN.CONGESTED) {
+ congestedNodesFromAck.add(targets[i]);
+ }
// Restart will not be treated differently unless it is
// the local node or the only one in the pipeline.
if (PipelineAck.isRestartOOBStatus(reply) &&
@@ -839,6 +856,18 @@ class DataStreamer extends Daemon {
}
}
+ if (!congestedNodesFromAck.isEmpty()) {
+ synchronized (congestedNodes) {
+ congestedNodes.clear();
+ congestedNodes.addAll(congestedNodesFromAck);
+ }
+ } else {
+ synchronized (congestedNodes) {
+ congestedNodes.clear();
+ lastCongestionBackoffTime = 0;
+ }
+ }
+
assert seqno != PipelineAck.UNKOWN_SEQNO :
"Ack for unknown seqno should be a failed ack: " + ack;
if (seqno == DFSPacket.HEART_BEAT_SEQNO) { // a heartbeat ack
@@ -1544,6 +1573,40 @@ class DataStreamer extends Daemon {
}
/**
+ * This function sleeps for a certain amount of time when the writing
+ * pipeline is congested. The function calculates the time based on a
+ * decorrelated filter.
+ *
+ * @see
+ * <a href="http://www.awsarchitectureblog.com/2015/03/backoff.html">
+ * http://www.awsarchitectureblog.com/2015/03/backoff.html</a>
+ */
+ private void backOffIfNecessary() throws InterruptedException {
+ int t = 0;
+ synchronized (congestedNodes) {
+ if (!congestedNodes.isEmpty()) {
+ StringBuilder sb = new StringBuilder("DataNode");
+ for (DatanodeInfo i : congestedNodes) {
+ sb.append(' ').append(i);
+ }
+ int range = Math.abs(lastCongestionBackoffTime * 3 -
+ CONGESTION_BACKOFF_MEAN_TIME_IN_MS);
+ int base = Math.min(lastCongestionBackoffTime * 3,
+ CONGESTION_BACKOFF_MEAN_TIME_IN_MS);
+ t = Math.min(CONGESTION_BACK_OFF_MAX_TIME_IN_MS,
+ (int)(base + Math.random() * range));
+ lastCongestionBackoffTime = t;
+ sb.append(" are congested. Backing off for ").append(t).append(" ms");
+ DFSClient.LOG.info(sb.toString());
+ congestedNodes.clear();
+ }
+ }
+ if (t != 0) {
+ Thread.sleep(t);
+ }
+ }
+
+ /**
* get the block this streamer is writing to
*
* @return the block this streamer is writing to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d4143f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
index 9bd4115..a811f39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
@@ -257,6 +257,10 @@ public class PipelineAck {
return StatusFormat.getStatus(header);
}
+ public static ECN getECNFromHeader(int header) {
+ return StatusFormat.getECN(header);
+ }
+
public static int setStatusForHeader(int old, Status status) {
return StatusFormat.setStatus(old, status);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d4143f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
index b47e7f1..a410e74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
@@ -17,20 +17,31 @@
*/
package org.apache.hadoop.hdfs;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.internal.util.reflection.Whitebox;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.when;
+
public class TestDFSOutputStream {
static MiniDFSCluster cluster;
@@ -100,6 +111,37 @@ public class TestDFSOutputStream {
Assert.assertTrue((Integer) field.get(dos) + 257 < packetSize);
}
+ @Test
+ public void testCongestionBackoff() throws IOException {
+ DFSClient.Conf dfsClientConf = mock(DFSClient.Conf.class);
+ DFSClient client = mock(DFSClient.class);
+ when(client.getConf()).thenReturn(dfsClientConf);
+ client.clientRunning = true;
+ DataStreamer stream = new DataStreamer(
+ mock(HdfsFileStatus.class),
+ mock(ExtendedBlock.class),
+ client,
+ "foo", null, null, null, null);
+
+ DataOutputStream blockStream = mock(DataOutputStream.class);
+ doThrow(new IOException()).when(blockStream).flush();
+ Whitebox.setInternalState(stream, "blockStream", blockStream);
+ Whitebox.setInternalState(stream, "stage",
+ BlockConstructionStage.PIPELINE_CLOSE);
+ @SuppressWarnings("unchecked")
+ LinkedList<DFSPacket> dataQueue = (LinkedList<DFSPacket>)
+ Whitebox.getInternalState(stream, "dataQueue");
+ @SuppressWarnings("unchecked")
+ ArrayList<DatanodeInfo> congestedNodes = (ArrayList<DatanodeInfo>)
+ Whitebox.getInternalState(stream, "congestedNodes");
+ congestedNodes.add(mock(DatanodeInfo.class));
+ DFSPacket packet = mock(DFSPacket.class);
+ when(packet.getTraceParents()).thenReturn(new long[] {});
+ dataQueue.add(packet);
+ stream.run();
+ Assert.assertTrue(congestedNodes.isEmpty());
+ }
+
@AfterClass
public static void tearDown() {
cluster.shutdown();
[50/50] [abbrv] hadoop git commit: HDFS-7999.
FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl lock for a
very long time (sinago via cmccabe)
Posted by zj...@apache.org.
HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl lock for a very long time (sinago via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8898d14c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8898d14c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8898d14c
Branch: refs/heads/YARN-2928
Commit: 8898d14c00981c48bdf6c575d6604e19adaca7f7
Parents: 37725e7
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Apr 6 08:54:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:17 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 77 +++++++++++++-------
2 files changed, 52 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8898d14c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6fafec8..52325a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1379,6 +1379,9 @@ Release 2.7.0 - UNRELEASED
HDFS-8051. FsVolumeList#addVolume should release volume reference if not
put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe)
+ HDFS-7999. FsDatasetImpl#createTemporary sometimes holds the FSDatasetImpl
+ lock for a very long time (sinago via cmccabe)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8898d14c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index f15f649..6bcbe5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1412,38 +1412,59 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public synchronized ReplicaHandler createTemporary(
+ public ReplicaHandler createTemporary(
StorageType storageType, ExtendedBlock b) throws IOException {
- ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId());
- if (replicaInfo != null) {
- if (replicaInfo.getGenerationStamp() < b.getGenerationStamp()
- && replicaInfo instanceof ReplicaInPipeline) {
- // Stop the previous writer
- ((ReplicaInPipeline)replicaInfo)
- .stopWriter(datanode.getDnConf().getXceiverStopTimeout());
- invalidate(b.getBlockPoolId(), new Block[]{replicaInfo});
- } else {
- throw new ReplicaAlreadyExistsException("Block " + b +
- " already exists in state " + replicaInfo.getState() +
- " and thus cannot be created.");
+ long startTimeMs = Time.monotonicNow();
+ long writerStopTimeoutMs = datanode.getDnConf().getXceiverStopTimeout();
+ ReplicaInfo lastFoundReplicaInfo = null;
+ do {
+ synchronized (this) {
+ ReplicaInfo currentReplicaInfo =
+ volumeMap.get(b.getBlockPoolId(), b.getBlockId());
+ if (currentReplicaInfo == lastFoundReplicaInfo) {
+ if (lastFoundReplicaInfo != null) {
+ invalidate(b.getBlockPoolId(), new Block[] { lastFoundReplicaInfo });
+ }
+ FsVolumeReference ref =
+ volumes.getNextVolume(storageType, b.getNumBytes());
+ FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
+ // create a temporary file to hold block in the designated volume
+ File f;
+ try {
+ f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock());
+ } catch (IOException e) {
+ IOUtils.cleanup(null, ref);
+ throw e;
+ }
+ ReplicaInPipeline newReplicaInfo =
+ new ReplicaInPipeline(b.getBlockId(), b.getGenerationStamp(), v,
+ f.getParentFile(), 0);
+ volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
+ return new ReplicaHandler(newReplicaInfo, ref);
+ } else {
+ if (!(currentReplicaInfo.getGenerationStamp() < b
+ .getGenerationStamp() && currentReplicaInfo instanceof ReplicaInPipeline)) {
+ throw new ReplicaAlreadyExistsException("Block " + b
+ + " already exists in state " + currentReplicaInfo.getState()
+ + " and thus cannot be created.");
+ }
+ lastFoundReplicaInfo = currentReplicaInfo;
+ }
}
- }
- FsVolumeReference ref = volumes.getNextVolume(storageType, b.getNumBytes());
- FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
- // create a temporary file to hold block in the designated volume
- File f;
- try {
- f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock());
- } catch (IOException e) {
- IOUtils.cleanup(null, ref);
- throw e;
- }
+ // Hang too long, just bail out. This is not supposed to happen.
+ long writerStopMs = Time.monotonicNow() - startTimeMs;
+ if (writerStopMs > writerStopTimeoutMs) {
+ LOG.warn("Unable to stop existing writer for block " + b + " after "
+ + writerStopMs + " miniseconds.");
+ throw new IOException("Unable to stop existing writer for block " + b
+ + " after " + writerStopMs + " miniseconds.");
+ }
- ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(),
- b.getGenerationStamp(), v, f.getParentFile(), 0);
- volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
- return new ReplicaHandler(newReplicaInfo, ref);
+ // Stop the previous writer
+ ((ReplicaInPipeline) lastFoundReplicaInfo)
+ .stopWriter(writerStopTimeoutMs);
+ } while (true);
}
/**
[18/50] [abbrv] hadoop git commit: HDFS-7922. ShortCircuitCache#close
is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
Posted by zj...@apache.org.
HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/093e0925
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/093e0925
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/093e0925
Branch: refs/heads/YARN-2928
Commit: 093e0925a4459bb07e4093a30455016a815734d5
Parents: f4afce0
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Apr 1 16:02:39 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/shortcircuit/ShortCircuitCache.java | 28 ++++++++++++++++++++
.../shortcircuit/TestShortCircuitCache.java | 2 +-
3 files changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/093e0925/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b5591e0..f265ead 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -409,6 +409,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6945. BlockManager should remove a block from excessReplicateMap and
decrement ExcessBlocks metric when the block is removed. (aajisaka)
+ HDFS-7922. ShortCircuitCache#close is not releasing
+ ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/093e0925/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index 73c52d5..d1ec3b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -916,6 +916,34 @@ public class ShortCircuitCache implements Closeable {
} finally {
lock.unlock();
}
+
+ releaserExecutor.shutdown();
+ cleanerExecutor.shutdown();
+ // wait for existing tasks to terminate
+ try {
+ if (!releaserExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+ LOG.error("Forcing SlotReleaserThreadPool to shutdown!");
+ releaserExecutor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ releaserExecutor.shutdownNow();
+ Thread.currentThread().interrupt();
+ LOG.error("Interrupted while waiting for SlotReleaserThreadPool "
+ + "to terminate", e);
+ }
+
+ // wait for existing tasks to terminate
+ try {
+ if (!cleanerExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
+ LOG.error("Forcing CleanerThreadPool to shutdown!");
+ cleanerExecutor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ cleanerExecutor.shutdownNow();
+ Thread.currentThread().interrupt();
+ LOG.error("Interrupted while waiting for CleanerThreadPool "
+ + "to terminate", e);
+ }
IOUtils.cleanup(LOG, shmManager);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/093e0925/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index 7daabd0..7d26dee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -203,7 +203,7 @@ public class TestShortCircuitCache {
cache.close();
}
- @Test(timeout=60000)
+ @Test(timeout=100000)
public void testExpiry() throws Exception {
final ShortCircuitCache cache =
new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0);
[41/50] [abbrv] hadoop git commit: HADOOP-11785. Reduce the number of
listStatus operation in distcp buildListing (Zoran Dimitrijevic via Colin P.
McCabe)
Posted by zj...@apache.org.
HADOOP-11785. Reduce the number of listStatus operation in distcp buildListing (Zoran Dimitrijevic via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb13303d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb13303d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb13303d
Branch: refs/heads/YARN-2928
Commit: fb13303de639de8b45c78e91d22c85774ad4252e
Parents: 4aff1e7
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Apr 3 14:08:25 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:16 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/tools/SimpleCopyListing.java | 41 +++++++++-----------
2 files changed, 21 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb13303d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7dcf8c1..d2d1181 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -483,6 +483,9 @@ Release 2.8.0 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-11785. Reduce the number of listStatus operation in distcp
+ buildListing (Zoran Dimitrijevic via Colin P. McCabe)
+
BUG FIXES
HADOOP-10027. *Compressor_deflateBytesDirect passes instance instead of
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb13303d/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index 6dc827a..e8a23aa 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -193,12 +193,12 @@ public class SimpleCopyListing extends CopyListing {
writeToFileListing(fileListWriter, sourceCopyListingStatus,
sourcePathRoot, options);
- if (isDirectoryAndNotEmpty(sourceFS, sourceStatus)) {
+ if (sourceStatus.isDirectory()) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Traversing non-empty source dir: " + sourceStatus.getPath());
+ LOG.debug("Traversing source dir: " + sourceStatus.getPath());
}
- traverseNonEmptyDirectory(fileListWriter, sourceStatus, sourcePathRoot,
- options);
+ traverseDirectory(fileListWriter, sourceFS, sourceStatus,
+ sourcePathRoot, options);
}
}
}
@@ -275,22 +275,17 @@ public class SimpleCopyListing extends CopyListing {
SequenceFile.Writer.compression(SequenceFile.CompressionType.NONE));
}
- private static boolean isDirectoryAndNotEmpty(FileSystem fileSystem,
- FileStatus fileStatus) throws IOException {
- return fileStatus.isDirectory() && getChildren(fileSystem, fileStatus).length > 0;
- }
-
private static FileStatus[] getChildren(FileSystem fileSystem,
FileStatus parent) throws IOException {
return fileSystem.listStatus(parent.getPath());
}
- private void traverseNonEmptyDirectory(SequenceFile.Writer fileListWriter,
- FileStatus sourceStatus,
- Path sourcePathRoot,
- DistCpOptions options)
- throws IOException {
- FileSystem sourceFS = sourcePathRoot.getFileSystem(getConf());
+ private void traverseDirectory(SequenceFile.Writer fileListWriter,
+ FileSystem sourceFS,
+ FileStatus sourceStatus,
+ Path sourcePathRoot,
+ DistCpOptions options)
+ throws IOException {
final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
final boolean preserveRawXattrs = options.shouldPreserveRawXattrs();
@@ -299,9 +294,9 @@ public class SimpleCopyListing extends CopyListing {
while (!pathStack.isEmpty()) {
for (FileStatus child: getChildren(sourceFS, pathStack.pop())) {
- if (LOG.isDebugEnabled())
- LOG.debug("Recording source-path: "
- + sourceStatus.getPath() + " for copy.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Recording source-path: " + child.getPath() + " for copy.");
+ }
CopyListingFileStatus childCopyListingStatus =
DistCpUtils.toCopyListingFileStatus(sourceFS, child,
preserveAcls && child.isDirectory(),
@@ -309,16 +304,16 @@ public class SimpleCopyListing extends CopyListing {
preserveRawXattrs && child.isDirectory());
writeToFileListing(fileListWriter, childCopyListingStatus,
sourcePathRoot, options);
- if (isDirectoryAndNotEmpty(sourceFS, child)) {
- if (LOG.isDebugEnabled())
- LOG.debug("Traversing non-empty source dir: "
- + sourceStatus.getPath());
+ if (child.isDirectory()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Traversing into source dir: " + child.getPath());
+ }
pathStack.push(child);
}
}
}
}
-
+
private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
CopyListingFileStatus fileStatus, Path sourcePathRoot,
DistCpOptions options) throws IOException {
[48/50] [abbrv] hadoop git commit:
TestFairScheduler.testContinuousScheduling fails Intermittently. Contributed
by Zhihai Xu.
Posted by zj...@apache.org.
TestFairScheduler.testContinuousScheduling fails Intermittently. Contributed by Zhihai Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37725e71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37725e71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37725e71
Branch: refs/heads/YARN-2928
Commit: 37725e71643f9dd6f68796736cc9c407ea437cb3
Parents: 0b09b40
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Mon Apr 6 20:13:21 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:17 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/scheduler/fair/TestFairScheduler.java | 7 ++-----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37725e71/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0c5aac8..5f20ca3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -201,6 +201,9 @@ Release 2.8.0 - UNRELEASED
YARN-3435. AM container to be allocated Appattempt AM container shown as null.
(Bibin A Chundatt via xgong)
+ YARN-2666. TestFairScheduler.testContinuousScheduling fails Intermittently.
+ (Zhihai Xu via ozawa)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37725e71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index b5bfb8c..98877e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3922,12 +3922,9 @@ public class TestFairScheduler extends FairSchedulerTestBase {
createResourceRequest(1024, 1, ResourceRequest.ANY, 2, 1, true);
ask.clear();
ask.add(request);
+ scheduler.stop();
scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
-
- // Wait until app gets resources
- while (app.getCurrentConsumption()
- .equals(Resources.createResource(1024, 1))) { }
-
+ scheduler.continuousSchedulingAttempt();
Assert.assertEquals(2048, app.getCurrentConsumption().getMemory());
Assert.assertEquals(2, app.getCurrentConsumption().getVirtualCores());
[39/50] [abbrv] hadoop git commit: MAPREDUCE-4844. Counters /
AbstractCounters have constant references not declared final. (Brahma Reddy
Battula via gera)
Posted by zj...@apache.org.
MAPREDUCE-4844. Counters / AbstractCounters have constant references not declared final. (Brahma Reddy Battula via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/def22772
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/def22772
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/def22772
Branch: refs/heads/YARN-2928
Commit: def2277201a7221c8937f25da13801fd56434c7b
Parents: 310c997
Author: Gera Shegalov <ge...@apache.org>
Authored: Fri Apr 3 10:24:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:16 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapred/Counters.java | 6 +++---
.../org/apache/hadoop/mapreduce/counters/AbstractCounters.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/def22772/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 160c49c..c1eb6c3 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -321,6 +321,9 @@ Release 2.8.0 - UNRELEASED
causes counter limits are not reset correctly.
(Zhihai Xu via harsh)
+ MAPREDUCE-4844. Counters / AbstractCounters have constant references not
+ declared final. (Brahma Reddy Battula via gera)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/def22772/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
index b9cb210..c9e0b9f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
@@ -62,9 +62,9 @@ import com.google.common.collect.Iterators;
public class Counters
extends AbstractCounters<Counters.Counter, Counters.Group> {
- public static int MAX_COUNTER_LIMIT = Limits.getCountersMax();
- public static int MAX_GROUP_LIMIT = Limits.getGroupsMax();
- private static HashMap<String, String> depricatedCounterMap =
+ public static final int MAX_COUNTER_LIMIT = Limits.getCountersMax();
+ public static final int MAX_GROUP_LIMIT = Limits.getGroupsMax();
+ private static final HashMap<String, String> depricatedCounterMap =
new HashMap<String, String>();
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/def22772/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
index dd81ebb..e6e74da 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
@@ -64,11 +64,11 @@ public abstract class AbstractCounters<C extends Counter,
/**
* A cache from enum values to the associated counter.
*/
- private Map<Enum<?>, C> cache = Maps.newIdentityHashMap();
+ private final Map<Enum<?>, C> cache = Maps.newIdentityHashMap();
//framework & fs groups
- private Map<String, G> fgroups = new ConcurrentSkipListMap<String, G>();
+ private final Map<String, G> fgroups = new ConcurrentSkipListMap<String, G>();
// other groups
- private Map<String, G> groups = new ConcurrentSkipListMap<String, G>();
+ private final Map<String, G> groups = new ConcurrentSkipListMap<String, G>();
private final CounterGroupFactory<C, G> groupFactory;
// For framework counter serialization without strings
[10/50] [abbrv] hadoop git commit: YARN-3428. Debug log resources to
be localized for a container. (kasha)
Posted by zj...@apache.org.
YARN-3428. Debug log resources to be localized for a container. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b62eb19e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b62eb19e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b62eb19e
Branch: refs/heads/YARN-2928
Commit: b62eb19ef02a23a766e4d8fa5172d57a36aec849
Parents: 0834fe3
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Mar 31 17:34:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:10 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../localizer/ResourceLocalizationService.java | 12 ++++++++++--
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b62eb19e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3cf9847..dab8116 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -145,6 +145,8 @@ Release 2.8.0 - UNRELEASED
YARN-3258. FairScheduler: Need to add more logging to investigate
allocations. (Anubhav Dhoot via ozawa)
+ YARN-3428. Debug log resources to be localized for a container. (kasha)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b62eb19e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index dd50ead..4236392 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -448,6 +448,10 @@ public class ResourceLocalizationService extends CompositeService
.getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Localizing " + req.getPath() +
+ " for container " + c.getContainerId());
+ }
}
}
}
@@ -456,10 +460,14 @@ public class ResourceLocalizationService extends CompositeService
ResourceRetentionSet retain =
new ResourceRetentionSet(delService, cacheTargetSize);
retain.addResources(publicRsrc);
- LOG.debug("Resource cleanup (public) " + retain);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resource cleanup (public) " + retain);
+ }
for (LocalResourcesTracker t : privateRsrc.values()) {
retain.addResources(t);
- LOG.debug("Resource cleanup " + t.getUser() + ":" + retain);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resource cleanup " + t.getUser() + ":" + retain);
+ }
}
//TODO Check if appRsrcs should also be added to the retention set.
}
[42/50] [abbrv] hadoop git commit: HDFS-8039. Fix
TestDebugAdmin#testRecoverLease and testVerfiyBlockChecksumCommand on
Windows. Contributed by Xiaoyu Yao.
Posted by zj...@apache.org.
HDFS-8039. Fix TestDebugAdmin#testRecoverLease and testVerfiyBlockChecksumCommand on Windows. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4aff1e7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4aff1e7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4aff1e7b
Branch: refs/heads/YARN-2928
Commit: 4aff1e7bf7866f547163a09b750a14a70fa0602e
Parents: def2277
Author: cnauroth <cn...@apache.org>
Authored: Fri Apr 3 10:44:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:16 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/tools/TestDebugAdmin.java | 15 ++++++++-------
2 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aff1e7b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 366d8fb..ae56f1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1370,6 +1370,9 @@ Release 2.7.0 - UNRELEASED
TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay
for Windows. (Xiaoyu Yao via cnauroth)
+ HDFS-8039. Fix TestDebugAdmin#testRecoverLease and
+ testVerifyBlockChecksumCommand on Windows. (Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aff1e7b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
index 44b6ba9..52b194d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
@@ -77,17 +77,18 @@ public class TestDebugAdmin {
System.setOut(oldOut);
IOUtils.closeStream(out);
}
- return "ret: " + ret + ", " + bytes.toString();
+ return "ret: " + ret + ", " +
+ bytes.toString().replaceAll(System.getProperty("line.separator"), "");
}
@Test(timeout = 60000)
public void testRecoverLease() throws Exception {
- assertEquals("ret: 1, You must supply a -path argument to recoverLease.\n",
+ assertEquals("ret: 1, You must supply a -path argument to recoverLease.",
runCmd(new String[]{"recoverLease", "-retries", "1"}));
FSDataOutputStream out = fs.create(new Path("/foo"));
out.write(123);
out.close();
- assertEquals("ret: 0, recoverLease SUCCEEDED on /foo\n",
+ assertEquals("ret: 0, recoverLease SUCCEEDED on /foo",
runCmd(new String[]{"recoverLease", "-path", "/foo"}));
}
@@ -98,18 +99,18 @@ public class TestDebugAdmin {
ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar"));
File blockFile = getBlockFile(fsd,
block.getBlockPoolId(), block.getLocalBlock());
- assertEquals("ret: 1, You must specify a meta file with -meta\n",
+ assertEquals("ret: 1, You must specify a meta file with -meta",
runCmd(new String[]{"verify", "-block", blockFile.getAbsolutePath()}));
File metaFile = getMetaFile(fsd,
block.getBlockPoolId(), block.getLocalBlock());
assertEquals("ret: 0, Checksum type: " +
- "DataChecksum(type=CRC32C, chunkSize=512)\n",
+ "DataChecksum(type=CRC32C, chunkSize=512)",
runCmd(new String[]{"verify",
"-meta", metaFile.getAbsolutePath()}));
assertEquals("ret: 0, Checksum type: " +
- "DataChecksum(type=CRC32C, chunkSize=512)\n" +
+ "DataChecksum(type=CRC32C, chunkSize=512)" +
"Checksum verification succeeded on block file " +
- blockFile.getAbsolutePath() + "\n",
+ blockFile.getAbsolutePath(),
runCmd(new String[]{"verify",
"-meta", metaFile.getAbsolutePath(),
"-block", blockFile.getAbsolutePath()})
[03/50] [abbrv] hadoop git commit: Reverted MAPREDUCE-6286,
MAPREDUCE-6199,
and MAPREDUCE-5875 from branch-2.7. Editing CHANGES.txt to reflect this.
Posted by zj...@apache.org.
Reverted MAPREDUCE-6286, MAPREDUCE-6199, and MAPREDUCE-5875 from branch-2.7. Editing CHANGES.txt to reflect this.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40a12828
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40a12828
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40a12828
Branch: refs/heads/YARN-2928
Commit: 40a1282880788e2dd2a82ad54838f357348117b3
Parents: 5b9d546
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Mar 31 13:29:20 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40a12828/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index b0367a7..f5d2d1a 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -311,6 +311,16 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6294. Remove an extra parameter described in Javadoc of
TockenCache. (Brahma Reddy Battula via ozawa)
+ MAPREDUCE-5875. Make Counter limits consistent across JobClient,
+ MRAppMaster, and YarnChild. (Gera Shegalov via kasha)
+
+ MAPREDUCE-6199. AbstractCounters are not reset completely on
+ deserialization (adhoot via rkanter)
+
+ MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which
+ causes counter limits are not reset correctly.
+ (Zhihai Xu via harsh)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -411,10 +421,6 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
- MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which
- causes counter limits are not reset correctly.
- (Zhihai Xu via harsh)
-
MAPREDUCE-6210. Use getApplicationAttemptId() instead of getApplicationId()
for logging AttemptId in RMContainerAllocator.java (Leitao Guo via aajisaka)
@@ -448,9 +454,6 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6045. need close the DataInputStream after open it in
TestMapReduce.java (zxu via rkanter)
- MAPREDUCE-6199. AbstractCounters are not reset completely on
- deserialization (adhoot via rkanter)
-
MAPREDUCE-6206. TestAggregatedTransferRate fails on non-US systems (Jens
Rabe via jlowe)
@@ -721,9 +724,6 @@ Release 2.6.0 - 2014-11-18
MAPREDUCE-6123. TestCombineFileInputFormat incorrectly starts 2
MiniDFSCluster instances. (cnauroth)
- MAPREDUCE-5875. Make Counter limits consistent across JobClient,
- MRAppMaster, and YarnChild. (Gera Shegalov via kasha)
-
MAPREDUCE-6125. TestContainerLauncherImpl sometimes fails (Mit Desai via
jlowe)
[02/50] [abbrv] hadoop git commit: HDFS-6945. BlockManager should
remove a block from excessReplicateMap and decrement ExcessBlocks metric when
the block is removed. (aajisaka)
Posted by zj...@apache.org.
HDFS-6945. BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71b43741
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71b43741
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71b43741
Branch: refs/heads/YARN-2928
Commit: 71b43741dde686725ac188ae91b14185989944a1
Parents: 40a1282
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Apr 1 09:07:28 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/BlockManager.java | 22 ++++++++++++++++++--
.../namenode/metrics/TestNameNodeMetrics.java | 9 ++++++--
3 files changed, 30 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f3537b0..4247ea6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -398,6 +398,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7997. The first non-existing xattr should also throw IOException.
(zhouyingchao via yliu)
+ HDFS-6945. BlockManager should remove a block from excessReplicateMap and
+ decrement ExcessBlocks metric when the block is removed. (aajisaka)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index f6e15a3..d9aee62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3351,8 +3351,7 @@ public class BlockManager {
// file already removes them from the block map below.
block.setNumBytes(BlockCommand.NO_ACK);
addToInvalidates(block);
- corruptReplicas.removeFromCorruptReplicasMap(block);
- blocksMap.removeBlock(block);
+ removeBlockFromMap(block);
// Remove the block from pendingReplications and neededReplications
pendingReplications.remove(block);
neededReplications.remove(block, UnderReplicatedBlocks.LEVEL);
@@ -3528,11 +3527,30 @@ public class BlockManager {
}
public void removeBlockFromMap(Block block) {
+ removeFromExcessReplicateMap(block);
blocksMap.removeBlock(block);
// If block is removed from blocksMap remove it from corruptReplicasMap
corruptReplicas.removeFromCorruptReplicasMap(block);
}
+ /**
+ * If a block is removed from blocksMap, remove it from excessReplicateMap.
+ */
+ private void removeFromExcessReplicateMap(Block block) {
+ for (DatanodeStorageInfo info : blocksMap.getStorages(block)) {
+ String uuid = info.getDatanodeDescriptor().getDatanodeUuid();
+ LightWeightLinkedSet<Block> excessReplicas = excessReplicateMap.get(uuid);
+ if (excessReplicas != null) {
+ if (excessReplicas.remove(block)) {
+ excessBlocksCount.decrementAndGet();
+ if (excessReplicas.isEmpty()) {
+ excessReplicateMap.remove(uuid);
+ }
+ }
+ }
+ }
+ }
+
public int getCapacity() {
return blocksMap.getCapacity();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71b43741/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index 64ea1e4..b390391 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -280,11 +280,16 @@ public class TestNameNodeMetrics {
public void testExcessBlocks() throws Exception {
Path file = getTestPath("testExcessBlocks");
createFile(file, 100, (short)2);
- long totalBlocks = 1;
NameNodeAdapter.setReplication(namesystem, file.toString(), (short)1);
MetricsRecordBuilder rb = getMetrics(NS_METRICS);
- assertGauge("ExcessBlocks", totalBlocks, rb);
+ assertGauge("ExcessBlocks", 1L, rb);
+
+ // verify ExcessBlocks metric is decremented and
+ // excessReplicateMap is cleared after deleting a file
fs.delete(file, true);
+ rb = getMetrics(NS_METRICS);
+ assertGauge("ExcessBlocks", 0L, rb);
+ assertTrue(bm.excessReplicateMap.isEmpty());
}
/** Test to ensure metrics reflects missing blocks */
[28/50] [abbrv] hadoop git commit: HADOOP-9805. Refactor
RawLocalFileSystem#rename for improved testability. Contributed by
Jean-Pierre Matsumoto.
Posted by zj...@apache.org.
HADOOP-9805. Refactor RawLocalFileSystem#rename for improved testability. Contributed by Jean-Pierre Matsumoto.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/503e4902
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/503e4902
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/503e4902
Branch: refs/heads/YARN-2928
Commit: 503e4902c88cbb6ddcfb423e8530c431dcdf0a8f
Parents: 88c1468
Author: cnauroth <cn...@apache.org>
Authored: Thu Apr 2 16:13:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:14 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/fs/RawLocalFileSystem.java | 36 ++++++++++-----
.../rawlocal/TestRawlocalContractRename.java | 47 ++++++++++++++++++++
3 files changed, 74 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/503e4902/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c92e378..260cc83 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -476,6 +476,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11660. Add support for hardware crc of HDFS checksums on ARM aarch64
architecture (Edward Nevill via Colin P. McCabe)
+ HADOOP-9805. Refactor RawLocalFileSystem#rename for improved testability.
+ (Jean-Pierre Matsumoto via cnauroth)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/503e4902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index d7866b8..52623b8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.io.nativeio.NativeIOException;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
@@ -347,11 +346,29 @@ public class RawLocalFileSystem extends FileSystem {
return true;
}
- // Enforce POSIX rename behavior that a source directory replaces an existing
- // destination if the destination is an empty directory. On most platforms,
- // this is already handled by the Java API call above. Some platforms
- // (notably Windows) do not provide this behavior, so the Java API call above
- // fails. Delete destination and attempt rename again.
+ // Else try POSIX style rename on Windows only
+ if (Shell.WINDOWS &&
+ handleEmptyDstDirectoryOnWindows(src, srcFile, dst, dstFile)) {
+ return true;
+ }
+
+ // The fallback behavior accomplishes the rename by a full copy.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Falling through to a copy of " + src + " to " + dst);
+ }
+ return FileUtil.copy(this, src, this, dst, true, getConf());
+ }
+
+ @VisibleForTesting
+ public final boolean handleEmptyDstDirectoryOnWindows(Path src, File srcFile,
+ Path dst, File dstFile) throws IOException {
+
+ // Enforce POSIX rename behavior that a source directory replaces an
+ // existing destination if the destination is an empty directory. On most
+ // platforms, this is already handled by the Java API call above. Some
+ // platforms (notably Windows) do not provide this behavior, so the Java API
+ // call renameTo(dstFile) fails. Delete destination and attempt rename
+ // again.
if (this.exists(dst)) {
FileStatus sdst = this.getFileStatus(dst);
if (sdst.isDirectory() && dstFile.list().length == 0) {
@@ -364,12 +381,7 @@ public class RawLocalFileSystem extends FileSystem {
}
}
}
-
- // The fallback behavior accomplishes the rename by a full copy.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Falling through to a copy of " + src + " to " + dst);
- }
- return FileUtil.copy(this, src, this, dst, true, getConf());
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/503e4902/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java
index 9e33b34..25611f1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java
@@ -19,8 +19,13 @@
package org.apache.hadoop.fs.contract.rawlocal;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
public class TestRawlocalContractRename extends AbstractContractRenameTest {
@@ -28,5 +33,47 @@ public class TestRawlocalContractRename extends AbstractContractRenameTest {
protected AbstractFSContract createContract(Configuration conf) {
return new RawlocalFSContract(conf);
}
+
+ /**
+ * Test fallback rename code <code>handleEmptyDstDirectoryOnWindows()</code>
+ * even on not Windows platform where the normal <code>File.renameTo()</code>
+ * is supposed to work well. This test has been added for HADOOP-9805.
+ *
+ * @see AbstractContractRenameTest#testRenameWithNonEmptySubDirPOSIX()
+ */
+ @Test
+ public void testRenameWithNonEmptySubDirPOSIX() throws Throwable {
+ final Path renameTestDir = path("testRenameWithNonEmptySubDir");
+ final Path srcDir = new Path(renameTestDir, "src1");
+ final Path srcSubDir = new Path(srcDir, "sub");
+ final Path finalDir = new Path(renameTestDir, "dest");
+ FileSystem fs = getFileSystem();
+ ContractTestUtils.rm(fs, renameTestDir, true, false);
+
+ fs.mkdirs(srcDir);
+ fs.mkdirs(finalDir);
+ ContractTestUtils.writeTextFile(fs, new Path(srcDir, "source.txt"),
+ "this is the file in src dir", false);
+ ContractTestUtils.writeTextFile(fs, new Path(srcSubDir, "subfile.txt"),
+ "this is the file in src/sub dir", false);
+
+ ContractTestUtils.assertPathExists(fs, "not created in src dir",
+ new Path(srcDir, "source.txt"));
+ ContractTestUtils.assertPathExists(fs, "not created in src/sub dir",
+ new Path(srcSubDir, "subfile.txt"));
+
+ RawLocalFileSystem rlfs = (RawLocalFileSystem) fs;
+ rlfs.handleEmptyDstDirectoryOnWindows(srcDir, rlfs.pathToFile(srcDir),
+ finalDir, rlfs.pathToFile(finalDir));
+
+ // Accept only POSIX rename behavior in this test
+ ContractTestUtils.assertPathExists(fs, "not renamed into dest dir",
+ new Path(finalDir, "source.txt"));
+ ContractTestUtils.assertPathExists(fs, "not renamed into dest/sub dir",
+ new Path(finalDir, "sub/subfile.txt"));
+
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted",
+ new Path(srcDir, "source.txt"));
+ }
}
[06/50] [abbrv] hadoop git commit: HDFS-7997. The first non-existing
xattr should also throw IOException. (zhouyingchao via yliu)
Posted by zj...@apache.org.
HDFS-7997. The first non-existing xattr should also throw IOException. (zhouyingchao via yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6916c77b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6916c77b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6916c77b
Branch: refs/heads/YARN-2928
Commit: 6916c77b9dae46d7e2ce2d0a3a233f33f044a469
Parents: 49db780
Author: yliu <yl...@apache.org>
Authored: Tue Mar 31 21:17:44 2015 +0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java | 3 ++-
.../hadoop-hdfs/src/test/resources/testXAttrConf.xml | 4 ++--
3 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6916c77b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0bea916..e8075e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -392,6 +392,9 @@ Release 2.8.0 - UNRELEASED
DatanodeDescriptor#updateHeartbeatState() (Brahma Reddy Battula via Colin
P. McCabe)
+ HDFS-7997. The first non-existing xattr should also throw IOException.
+ (zhouyingchao via yliu)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6916c77b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 45e63f2..d5c9124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -108,7 +108,8 @@ class FSDirXAttrOp {
return filteredAll;
}
if (filteredAll == null || filteredAll.isEmpty()) {
- return null;
+ throw new IOException(
+ "At least one of the attributes provided was not found.");
}
List<XAttr> toGet = Lists.newArrayListWithCapacity(xAttrs.size());
for (XAttr xAttr : xAttrs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6916c77b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml
index 9c66cba..c2e836c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml
@@ -420,8 +420,8 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>ExactComparator</type>
- <expected-output># file: /file1#LF#</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>At least one of the attributes provided was not found</expected-output>
</comparator>
</comparators>
</test>
[16/50] [abbrv] hadoop git commit: Add the missing files for
HDFS-8009.
Posted by zj...@apache.org.
Add the missing files for HDFS-8009.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/777f01b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/777f01b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/777f01b5
Branch: refs/heads/YARN-2928
Commit: 777f01b5defa1b53cde21042f5b56a22175e5a87
Parents: 0d7c075
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Apr 1 10:58:32 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:11 2015 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/TestDataNodeECN.java | 45 ++++++++++++++++++++
1 file changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/777f01b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java
new file mode 100644
index 0000000..b994386
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeECN.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestDataNodeECN {
+ @Test
+ public void testECNFlag() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED, true);
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ PipelineAck.ECN ecn = cluster.getDataNodes().get(0).getECN();
+ Assert.assertNotEquals(PipelineAck.ECN.DISABLED, ecn);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+}
[25/50] [abbrv] hadoop git commit: HDFS-8035. Move
checkBlocksProperlyReplicated() in FSNamesystem to BlockManager. Contributed
by Haohui Mai.
Posted by zj...@apache.org.
HDFS-8035. Move checkBlocksProperlyReplicated() in FSNamesystem to BlockManager. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d92ba156
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d92ba156
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d92ba156
Branch: refs/heads/YARN-2928
Commit: d92ba1569348218f2f2b72cd144294d921cff80d
Parents: 0a1dcd1
Author: Haohui Mai <wh...@apache.org>
Authored: Thu Apr 2 11:36:51 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:13 2015 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 21 +++++++++
.../hdfs/server/namenode/FSNamesystem.java | 49 ++++----------------
2 files changed, 30 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d92ba156/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index d9aee62..524afa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3408,6 +3408,27 @@ public class BlockManager {
}
}
+ /**
+ * Check that the indicated blocks are present and
+ * replicated.
+ */
+ public boolean checkBlocksProperlyReplicated(
+ String src, BlockInfoContiguous[] blocks) {
+ for (BlockInfoContiguous b: blocks) {
+ if (!b.isComplete()) {
+ final BlockInfoContiguousUnderConstruction uc =
+ (BlockInfoContiguousUnderConstruction)b;
+ final int numNodes = b.numNodes();
+ LOG.info("BLOCK* " + b + " is not COMPLETE (ucState = "
+ + uc.getBlockUCState() + ", replication# = " + numNodes
+ + (numNodes < minReplication ? " < ": " >= ")
+ + " minimum = " + minReplication + ") in file " + src);
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* @return 0 if the block is not found;
* otherwise, return the replication factor of the block.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d92ba156/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 59e1900..a77c382 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3525,20 +3525,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @param targets target datanodes where replicas of the new block is placed
* @throws QuotaExceededException If addition of block exceeds space quota
*/
- BlockInfoContiguous saveAllocatedBlock(String src, INodesInPath inodesInPath,
+ private void saveAllocatedBlock(String src, INodesInPath inodesInPath,
Block newBlock, DatanodeStorageInfo[] targets)
- throws IOException {
+ throws IOException {
assert hasWriteLock();
BlockInfoContiguous b = dir.addBlock(src, inodesInPath, newBlock, targets);
NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
- return b;
}
/**
* Create new block with a unique block id and a new generation stamp.
*/
- Block createNewBlock() throws IOException {
+ private Block createNewBlock() throws IOException {
assert hasWriteLock();
Block b = new Block(nextBlockId(), 0, 0);
// Increment the generation stamp for every new block.
@@ -3555,51 +3554,21 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
readLock();
try {
if (checkall) {
- // check all blocks of the file.
- for (BlockInfoContiguous block: v.getBlocks()) {
- if (!isCompleteBlock(src, block, blockManager.minReplication)) {
- return false;
- }
- }
+ return blockManager.checkBlocksProperlyReplicated(src, v
+ .getBlocks());
} else {
// check the penultimate block of this file
BlockInfoContiguous b = v.getPenultimateBlock();
- if (b != null
- && !isCompleteBlock(src, b, blockManager.minReplication)) {
- return false;
- }
+ return b == null ||
+ blockManager.checkBlocksProperlyReplicated(
+ src, new BlockInfoContiguous[] { b });
}
- return true;
} finally {
readUnlock();
}
}
- private static boolean isCompleteBlock(String src, BlockInfoContiguous b, int minRepl) {
- if (!b.isComplete()) {
- final BlockInfoContiguousUnderConstruction uc = (BlockInfoContiguousUnderConstruction)b;
- final int numNodes = b.numNodes();
- LOG.info("BLOCK* " + b + " is not COMPLETE (ucState = "
- + uc.getBlockUCState() + ", replication# = " + numNodes
- + (numNodes < minRepl? " < ": " >= ")
- + " minimum = " + minRepl + ") in file " + src);
- return false;
- }
- return true;
- }
-
- ////////////////////////////////////////////////////////////////
- // Here's how to handle block-copy failure during client write:
- // -- As usual, the client's write should result in a streaming
- // backup write to a k-machine sequence.
- // -- If one of the backup machines fails, no worries. Fail silently.
- // -- Before client is allowed to close and finalize file, make sure
- // that the blocks are backed up. Namenode may have to issue specific backup
- // commands to make up for earlier datanode failures. Once all copies
- // are made, edit namespace and return to client.
- ////////////////////////////////////////////////////////////////
-
- /**
+ /**
* Change the indicated filename.
* @deprecated Use {@link #renameTo(String, String, boolean,
* Options.Rename...)} instead.
[27/50] [abbrv] hadoop git commit: HDFS-7954.
TestBalancer#testBalancerWithPinnedBlocks should not be executed on Windows.
Contributed by Xiaoyu Yao
Posted by zj...@apache.org.
HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed on Windows. Contributed by Xiaoyu Yao
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a1dcd17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a1dcd17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a1dcd17
Branch: refs/heads/YARN-2928
Commit: 0a1dcd17cc585d145ad4ee1e84ac3d3267dd105c
Parents: b98f637
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Apr 2 11:11:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:13 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/balancer/TestBalancer.java | 6 ++++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a1dcd17/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0c66309..85c013e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1360,6 +1360,9 @@ Release 2.7.0 - UNRELEASED
HDFS-8001 RpcProgramNfs3 : wrong parsing of dfs.blocksize
(Remi Catherinot via brandonli)
+ HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed
+ on Windows. (Xiaoyu Yao via szetszwo)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a1dcd17/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 8b2d11e..9aea283 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.IOException;
@@ -309,6 +310,11 @@ public class TestBalancer {
*/
@Test(timeout=100000)
public void testBalancerWithPinnedBlocks() throws Exception {
+ // This test assumes stick-bit based block pin mechanism available only
+ // in Linux/Unix. It can be unblocked on Windows when HDFS-7759 is ready to
+ // provide a different mechanism for Windows.
+ assumeTrue(!Path.WINDOWS);
+
final Configuration conf = new HdfsConfiguration();
initConf(conf);
conf.setBoolean(DFS_DATANODE_BLOCK_PINNING_ENABLED, true);
[20/50] [abbrv] hadoop git commit: HDFS-8036. Use snapshot path as
source when using snapshot diff report in DistCp. Contributed by Jing Zhao.
Posted by zj...@apache.org.
HDFS-8036. Use snapshot path as source when using snapshot diff report in DistCp. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61487d36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61487d36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61487d36
Branch: refs/heads/YARN-2928
Commit: 61487d3642b69381c99b1463fce176063de15726
Parents: 093e092
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Apr 1 16:50:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/tools/DistCpSync.java | 21 +++++++++-
.../hadoop/tools/mapred/CopyCommitter.java | 3 +-
.../org/apache/hadoop/tools/TestDistCpSync.java | 40 +++++++++++++++++++-
4 files changed, 63 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f265ead..1d9e200 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1345,6 +1345,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7748. Separate ECN flags from the Status in the DataTransferPipelineAck.
(Anu Engineer and Haohui Mai via wheat9)
+ HDFS-8036. Use snapshot path as source when using snapshot diff report in
+ DistCp. (Jing Zhao via wheat9)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
index 26d7eb4..8e71b6f 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import java.io.IOException;
@@ -86,6 +87,22 @@ class DistCpSync {
} finally {
deleteTargetTmpDir(targetFs, tmpDir);
// TODO: since we have tmp directory, we can support "undo" with failures
+ // set the source path using the snapshot path
+ inputOptions.setSourcePaths(Arrays.asList(getSourceSnapshotPath(sourceDir,
+ inputOptions.getToSnapshot())));
+ }
+ }
+
+ private static String getSnapshotName(String name) {
+ return Path.CUR_DIR.equals(name) ? "" : name;
+ }
+
+ private static Path getSourceSnapshotPath(Path sourceDir, String snapshotName) {
+ if (Path.CUR_DIR.equals(snapshotName)) {
+ return sourceDir;
+ } else {
+ return new Path(sourceDir,
+ HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + snapshotName);
}
}
@@ -136,8 +153,10 @@ class DistCpSync {
static DiffInfo[] getDiffs(DistCpOptions inputOptions,
DistributedFileSystem fs, Path sourceDir, Path targetDir) {
try {
+ final String from = getSnapshotName(inputOptions.getFromSnapshot());
+ final String to = getSnapshotName(inputOptions.getToSnapshot());
SnapshotDiffReport sourceDiff = fs.getSnapshotDiffReport(sourceDir,
- inputOptions.getFromSnapshot(), inputOptions.getToSnapshot());
+ from, to);
return DiffInfo.getDiffs(sourceDiff, targetDir);
} catch (IOException e) {
DistCp.LOG.warn("Failed to compute snapshot diff on " + sourceDir, e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 9ec57f4..2b1e510 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -90,8 +90,7 @@ public class CopyCommitter extends FileOutputCommitter {
}
try {
- if (conf.getBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false)
- && !(conf.getBoolean(DistCpConstants.CONF_LABEL_DIFF, false))) {
+ if (conf.getBoolean(DistCpConstants.CONF_LABEL_DELETE_MISSING, false)) {
deleteMissing(conf);
} else if (conf.getBoolean(DistCpConstants.CONF_LABEL_ATOMIC_COPY, false)) {
commitData(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61487d36/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
index 7d5dad0..75d1de5 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
@@ -97,6 +98,8 @@ public class TestDistCpSync {
dfs.createSnapshot(source, "s2");
dfs.createSnapshot(target, "s1");
Assert.assertTrue(DistCpSync.sync(options, conf));
+ // reset source paths in options
+ options.setSourcePaths(Arrays.asList(source));
// changes have been made in target
final Path subTarget = new Path(target, "sub");
@@ -183,9 +186,21 @@ public class TestDistCpSync {
changeData(source);
dfs.createSnapshot(source, "s2");
+ // before sync, make some further changes on source. this should not affect
+ // the later distcp since we're copying (s2-s1) to target
+ final Path toDelete = new Path(source, "foo/d1/foo/f1");
+ dfs.delete(toDelete, true);
+ final Path newdir = new Path(source, "foo/d1/foo/newdir");
+ dfs.mkdirs(newdir);
+
// do the sync
Assert.assertTrue(DistCpSync.sync(options, conf));
+ // make sure the source path has been updated to the snapshot path
+ final Path spath = new Path(source,
+ HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
+ Assert.assertEquals(spath, options.getSourcePaths().get(0));
+
// build copy listing
final Path listingPath = new Path("/tmp/META/fileList.seq");
CopyListing listing = new GlobbedCopyListing(conf, new Credentials());
@@ -209,7 +224,7 @@ public class TestDistCpSync {
.getCounter(CopyMapper.Counter.BYTESCOPIED).getValue());
// verify the source and target now has the same structure
- verifyCopy(dfs.getFileStatus(source), dfs.getFileStatus(target), false);
+ verifyCopy(dfs.getFileStatus(spath), dfs.getFileStatus(target), false);
}
private Map<Text, CopyListingFileStatus> getListing(Path listingPath)
@@ -248,6 +263,29 @@ public class TestDistCpSync {
}
}
+ /**
+ * Similar test with testSync, but the "to" snapshot is specified as "."
+ * @throws Exception
+ */
+ @Test
+ public void testSyncWithCurrent() throws Exception {
+ options.setUseDiff(true, "s1", ".");
+ initData(source);
+ initData(target);
+ dfs.allowSnapshot(source);
+ dfs.allowSnapshot(target);
+ dfs.createSnapshot(source, "s1");
+ dfs.createSnapshot(target, "s1");
+
+ // make changes under source
+ changeData(source);
+
+ // do the sync
+ Assert.assertTrue(DistCpSync.sync(options, conf));
+ // make sure the source path is still unchanged
+ Assert.assertEquals(source, options.getSourcePaths().get(0));
+ }
+
private void initData2(Path dir) throws Exception {
final Path test = new Path(dir, "test");
final Path foo = new Path(dir, "foo");
[24/50] [abbrv] hadoop git commit: HADOOP-11731. Rework the changelog
and releasenotes (aw)
Posted by zj...@apache.org.
HADOOP-11731. Rework the changelog and releasenotes (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e6b61eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e6b61eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e6b61eb
Branch: refs/heads/YARN-2928
Commit: 0e6b61ebe68350d7c4f1d5eaeb97e8977eab0b2f
Parents: ccf3fd1
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Apr 1 17:52:22 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:13 2015 -0700
----------------------------------------------------------------------
BUILDING.txt | 7 +-
dev-support/releasedocmaker.py | 460 +++++++++++++++++++
dev-support/relnotes.py | 274 -----------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-common/pom.xml | 51 ++
hadoop-project/src/site/site.xml | 6 +-
6 files changed, 518 insertions(+), 282 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 02b8610..f3b6853 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -73,7 +73,7 @@ Where to run Maven from?
----------------------------------------------------------------------------------
Maven build goals:
- * Clean : mvn clean
+ * Clean : mvn clean [-Preleasedocs]
* Compile : mvn compile [-Pnative]
* Run tests : mvn test [-Pnative]
* Create JAR : mvn package
@@ -84,7 +84,7 @@ Maven build goals:
* Run clover : mvn test -Pclover [-DcloverLicenseLocation=${user.name}/.clover.license]
* Run Rat : mvn apache-rat:check
* Build javadocs : mvn javadoc:javadoc
- * Build distribution : mvn package [-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar]
+ * Build distribution : mvn package [-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar][-Preleasedocs]
* Change Hadoop version : mvn versions:set -DnewVersion=NEWVERSION
Build options:
@@ -93,6 +93,7 @@ Maven build goals:
* Use -Pdocs to generate & bundle the documentation in the distribution (using -Pdist)
* Use -Psrc to create a project source TAR.GZ
* Use -Dtar to create a TAR with the distribution (using -Pdist)
+ * Use -Preleasedocs to include the changelog and release docs (requires Internet connectivity)
Snappy build options:
@@ -203,7 +204,7 @@ Create source and binary distributions with native code and documentation:
Create a local staging version of the website (in /tmp/hadoop-site)
- $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
+ $ mvn clean site -Preleasedocs; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
----------------------------------------------------------------------------------
Installing Hadoop
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
new file mode 100755
index 0000000..b00c1a7
--- /dev/null
+++ b/dev-support/releasedocmaker.py
@@ -0,0 +1,460 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from glob import glob
+from optparse import OptionParser
+import os
+import re
+import sys
+import urllib
+try:
+ import json
+except ImportError:
+ import simplejson as json
+
+releaseVersion={}
+namePattern = re.compile(r' \([0-9]+\)')
+
+def clean(str):
+ return tableclean(re.sub(namePattern, "", str))
+
+def formatComponents(str):
+ str = re.sub(namePattern, '', str).replace("'", "")
+ if str != "":
+ ret = str
+ else:
+ # some markdown parsers don't like empty tables
+ ret = "."
+ return clean(ret)
+
+# convert to utf-8
+# protect some known md metachars
+# or chars that screw up doxia
+def tableclean(str):
+ str=str.encode('utf-8')
+ str=str.replace("_","\_")
+ str=str.replace("\r","")
+ str=str.rstrip()
+ return str
+
+# same thing as tableclean,
+# except table metachars are also
+# escaped as well as more
+# things we don't want doxia to
+# screw up
+def notableclean(str):
+ str=tableclean(str)
+ str=str.replace("|","\|")
+ str=str.replace("<","\<")
+ str=str.replace(">","\>")
+ str=str.rstrip()
+ return str
+
+def mstr(obj):
+ if (obj == None):
+ return ""
+ return unicode(obj)
+
+def buildindex(master):
+ versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
+ with open("index.md","w") as indexfile:
+ for v in versions:
+ indexfile.write("* Apache Hadoop v%s\n" % (v))
+ for k in ("Changes","Release Notes"):
+ indexfile.write(" * %s\n" %(k))
+ indexfile.write(" * [Combined %s](%s/%s.%s.html)\n" \
+ % (k,v,k.upper().replace(" ",""),v))
+ if not master:
+ indexfile.write(" * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \
+ % (k,v,k.upper().replace(" ",""),v))
+ for p in ("HDFS","MapReduce","YARN"):
+ indexfile.write(" * [%s %s](%s/%s.%s.%s.html)\n" \
+ % (p,k,v,k.upper().replace(" ",""),p.upper(),v))
+ indexfile.close()
+
+class Version:
+ """Represents a version number"""
+ def __init__(self, data):
+ self.mod = False
+ self.data = data
+ found = re.match('^((\d+)(\.\d+)*).*$', data)
+ if (found):
+ self.parts = [ int(p) for p in found.group(1).split('.') ]
+ else:
+ self.parts = []
+ # backfill version with zeroes if missing parts
+ self.parts.extend((0,) * (3 - len(self.parts)))
+
+ def __str__(self):
+ if (self.mod):
+ return '.'.join([ str(p) for p in self.parts ])
+ return self.data
+
+ def __cmp__(self, other):
+ return cmp(self.parts, other.parts)
+
+class Jira:
+ """A single JIRA"""
+
+ def __init__(self, data, parent):
+ self.key = data['key']
+ self.fields = data['fields']
+ self.parent = parent
+ self.notes = None
+ self.incompat = None
+ self.reviewed = None
+
+ def getId(self):
+ return mstr(self.key)
+
+ def getDescription(self):
+ return mstr(self.fields['description'])
+
+ def getReleaseNote(self):
+ if (self.notes == None):
+ field = self.parent.fieldIdMap['Release Note']
+ if (self.fields.has_key(field)):
+ self.notes=mstr(self.fields[field])
+ else:
+ self.notes=self.getDescription()
+ return self.notes
+
+ def getPriority(self):
+ ret = ""
+ pri = self.fields['priority']
+ if(pri != None):
+ ret = pri['name']
+ return mstr(ret)
+
+ def getAssignee(self):
+ ret = ""
+ mid = self.fields['assignee']
+ if(mid != None):
+ ret = mid['displayName']
+ return mstr(ret)
+
+ def getComponents(self):
+ if (len(self.fields['components'])>0):
+ return ", ".join([ comp['name'] for comp in self.fields['components'] ])
+ else:
+ return ""
+
+ def getSummary(self):
+ return self.fields['summary']
+
+ def getType(self):
+ ret = ""
+ mid = self.fields['issuetype']
+ if(mid != None):
+ ret = mid['name']
+ return mstr(ret)
+
+ def getReporter(self):
+ ret = ""
+ mid = self.fields['reporter']
+ if(mid != None):
+ ret = mid['displayName']
+ return mstr(ret)
+
+ def getProject(self):
+ ret = ""
+ mid = self.fields['project']
+ if(mid != None):
+ ret = mid['key']
+ return mstr(ret)
+
+ def __cmp__(self,other):
+ selfsplit=self.getId().split('-')
+ othersplit=other.getId().split('-')
+ v1=cmp(selfsplit[0],othersplit[0])
+ if (v1!=0):
+ return v1
+ else:
+ if selfsplit[1] < othersplit[1]:
+ return True
+ elif selfsplit[1] > othersplit[1]:
+ return False
+ return False
+
+ def getIncompatibleChange(self):
+ if (self.incompat == None):
+ field = self.parent.fieldIdMap['Hadoop Flags']
+ self.reviewed=False
+ self.incompat=False
+ if (self.fields.has_key(field)):
+ if self.fields[field]:
+ for hf in self.fields[field]:
+ if hf['value'] == "Incompatible change":
+ self.incompat=True
+ if hf['value'] == "Reviewed":
+ self.reviewed=True
+ return self.incompat
+
+ def getReleaseDate(self,version):
+ for j in range(len(self.fields['fixVersions'])):
+ if self.fields['fixVersions'][j]==version:
+ return(self.fields['fixVersions'][j]['releaseDate'])
+ return None
+
+class JiraIter:
+ """An Iterator of JIRAs"""
+
+ def __init__(self, versions):
+ self.versions = versions
+
+ resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
+ data = json.loads(resp.read())
+
+ self.fieldIdMap = {}
+ for part in data:
+ self.fieldIdMap[part['name']] = part['id']
+
+ self.jiras = []
+ at=0
+ end=1
+ count=100
+ while (at < end):
+ params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
+ resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
+ data = json.loads(resp.read())
+ if (data.has_key('errorMessages')):
+ raise Exception(data['errorMessages'])
+ at = data['startAt'] + data['maxResults']
+ end = data['total']
+ self.jiras.extend(data['issues'])
+
+ needaversion=False
+ for j in versions:
+ v=str(j).replace("-SNAPSHOT","")
+ if v not in releaseVersion:
+ needaversion=True
+
+ if needaversion is True:
+ for i in range(len(data['issues'])):
+ for j in range(len(data['issues'][i]['fields']['fixVersions'])):
+ if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
+ releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
+ data['issues'][i]['fields']['fixVersions'][j]['releaseDate']
+
+ self.iter = self.jiras.__iter__()
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ data = self.iter.next()
+ j = Jira(data, self)
+ return j
+
+class Outputs:
+ """Several different files to output to at the same time"""
+
+ def __init__(self, base_file_name, file_name_pattern, keys, params={}):
+ self.params = params
+ self.base = open(base_file_name%params, 'w')
+ self.others = {}
+ for key in keys:
+ both = dict(params)
+ both['key'] = key
+ self.others[key] = open(file_name_pattern%both, 'w')
+
+ def writeAll(self, pattern):
+ both = dict(self.params)
+ both['key'] = ''
+ self.base.write(pattern%both)
+ for key in self.others.keys():
+ both = dict(self.params)
+ both['key'] = key
+ self.others[key].write(pattern%both)
+
+ def writeKeyRaw(self, key, str):
+ self.base.write(str)
+ if (self.others.has_key(key)):
+ self.others[key].write(str)
+
+ def close(self):
+ self.base.close()
+ for fd in self.others.values():
+ fd.close()
+
+ def writeList(self, mylist):
+ for jira in sorted(mylist):
+ line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' \
+ % (notableclean(jira.getId()), notableclean(jira.getId()),
+ notableclean(jira.getSummary()),
+ notableclean(jira.getPriority()),
+ formatComponents(jira.getComponents()),
+ notableclean(jira.getReporter()),
+ notableclean(jira.getAssignee()))
+ self.writeKeyRaw(jira.getProject(), line)
+
+def main():
+ parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]",
+ epilog=
+ "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
+ " named after the highest version provided.")
+ parser.add_option("-v", "--version", dest="versions",
+ action="append", type="string",
+ help="versions in JIRA to include in releasenotes", metavar="VERSION")
+ parser.add_option("-m","--master", dest="master", action="store_true",
+ help="only create the master, merged project files")
+ parser.add_option("-i","--index", dest="index", action="store_true",
+ help="build an index file")
+ (options, args) = parser.parse_args()
+
+ if (options.versions == None):
+ options.versions = []
+
+ if (len(args) > 2):
+ options.versions.append(args[2])
+
+ if (len(options.versions) <= 0):
+ parser.error("At least one version needs to be supplied")
+
+ versions = [ Version(v) for v in options.versions ];
+ versions.sort();
+
+ maxVersion = str(versions[-1])
+
+ jlist = JiraIter(versions)
+ version = maxVersion
+
+ if version in releaseVersion:
+ reldate=releaseVersion[version]
+ else:
+ reldate="Unreleased"
+
+ if not os.path.exists(version):
+ os.mkdir(version)
+
+ if options.master:
+ reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
+ "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
+ [], {"ver":maxVersion, "date":reldate})
+ choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
+ "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
+ [], {"ver":maxVersion, "date":reldate})
+ else:
+ reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
+ "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
+ ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
+ choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
+ "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
+ ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
+
+ relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
+ 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
+
+ chhead = '# Hadoop Changelog\n\n' \
+ '## Release %(ver)s - %(date)s\n'\
+ '\n'
+
+ reloutputs.writeAll(relhead)
+ choutputs.writeAll(chhead)
+
+ incompatlist=[]
+ buglist=[]
+ improvementlist=[]
+ newfeaturelist=[]
+ subtasklist=[]
+ tasklist=[]
+ testlist=[]
+ otherlist=[]
+
+ for jira in sorted(jlist):
+ if jira.getIncompatibleChange():
+ incompatlist.append(jira)
+ elif jira.getType() == "Bug":
+ buglist.append(jira)
+ elif jira.getType() == "Improvement":
+ improvementlist.append(jira)
+ elif jira.getType() == "New Feature":
+ newfeaturelist.append(jira)
+ elif jira.getType() == "Sub-task":
+ subtasklist.append(jira)
+ elif jira.getType() == "Task":
+ tasklist.append(jira)
+ elif jira.getType() == "Test":
+ testlist.append(jira)
+ else:
+ otherlist.append(jira)
+
+ line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
+ % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
+ notableclean(jira.getSummary()))
+
+ if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
+ reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
+ reloutputs.writeKeyRaw(jira.getProject(), line)
+ line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
+ print 'WARNING: incompatible change %s lacks release notes.' % (notableclean(jira.getId()))
+ reloutputs.writeKeyRaw(jira.getProject(), line)
+
+ if (len(jira.getReleaseNote())>0):
+ reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
+ reloutputs.writeKeyRaw(jira.getProject(), line)
+ line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
+ reloutputs.writeKeyRaw(jira.getProject(), line)
+
+ reloutputs.writeAll("\n\n")
+ reloutputs.close()
+
+ choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(incompatlist)
+
+ choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(newfeaturelist)
+
+ choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(improvementlist)
+
+ choutputs.writeAll("\n\n### BUG FIXES:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(buglist)
+
+ choutputs.writeAll("\n\n### TESTS:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(testlist)
+
+ choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(subtasklist)
+
+ choutputs.writeAll("\n\n### OTHER:\n\n")
+ choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
+ choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
+ choutputs.writeList(otherlist)
+ choutputs.writeList(tasklist)
+
+ choutputs.writeAll("\n\n")
+ choutputs.close()
+
+ if options.index:
+ buildindex(options.master)
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/dev-support/relnotes.py
----------------------------------------------------------------------
diff --git a/dev-support/relnotes.py b/dev-support/relnotes.py
deleted file mode 100644
index 57d48a4..0000000
--- a/dev-support/relnotes.py
+++ /dev/null
@@ -1,274 +0,0 @@
-#!/usr/bin/python
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import sys
-from optparse import OptionParser
-import httplib
-import urllib
-import cgi
-try:
- import json
-except ImportError:
- import simplejson as json
-
-
-namePattern = re.compile(r' \([0-9]+\)')
-
-def clean(str):
- return quoteHtml(re.sub(namePattern, "", str))
-
-def formatComponents(str):
- str = re.sub(namePattern, '', str).replace("'", "")
- if str != "":
- ret = "(" + str + ")"
- else:
- ret = ""
- return quoteHtml(ret)
-
-def quoteHtml(str):
- return cgi.escape(str).encode('ascii', 'xmlcharrefreplace')
-
-def mstr(obj):
- if (obj == None):
- return ""
- return unicode(obj)
-
-class Version:
- """Represents a version number"""
- def __init__(self, data):
- self.mod = False
- self.data = data
- found = re.match('^((\d+)(\.\d+)*).*$', data)
- if (found):
- self.parts = [ int(p) for p in found.group(1).split('.') ]
- else:
- self.parts = []
- # backfill version with zeroes if missing parts
- self.parts.extend((0,) * (3 - len(self.parts)))
-
- def decBugFix(self):
- self.mod = True
- self.parts[2] -= 1
- return self
-
- def __str__(self):
- if (self.mod):
- return '.'.join([ str(p) for p in self.parts ])
- return self.data
-
- def __cmp__(self, other):
- return cmp(self.parts, other.parts)
-
-class Jira:
- """A single JIRA"""
-
- def __init__(self, data, parent):
- self.key = data['key']
- self.fields = data['fields']
- self.parent = parent
- self.notes = None
-
- def getId(self):
- return mstr(self.key)
-
- def getDescription(self):
- return mstr(self.fields['description'])
-
- def getReleaseNote(self):
- if (self.notes == None):
- field = self.parent.fieldIdMap['Release Note']
- if (self.fields.has_key(field)):
- self.notes=mstr(self.fields[field])
- else:
- self.notes=self.getDescription()
- return self.notes
-
- def getPriority(self):
- ret = ""
- pri = self.fields['priority']
- if(pri != None):
- ret = pri['name']
- return mstr(ret)
-
- def getAssignee(self):
- ret = ""
- mid = self.fields['assignee']
- if(mid != None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getComponents(self):
- return " , ".join([ comp['name'] for comp in self.fields['components'] ])
-
- def getSummary(self):
- return self.fields['summary']
-
- def getType(self):
- ret = ""
- mid = self.fields['issuetype']
- if(mid != None):
- ret = mid['name']
- return mstr(ret)
-
- def getReporter(self):
- ret = ""
- mid = self.fields['reporter']
- if(mid != None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getProject(self):
- ret = ""
- mid = self.fields['project']
- if(mid != None):
- ret = mid['key']
- return mstr(ret)
-
-
-
-class JiraIter:
- """An Iterator of JIRAs"""
-
- def __init__(self, versions):
- self.versions = versions
-
- resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
- data = json.loads(resp.read())
-
- self.fieldIdMap = {}
- for part in data:
- self.fieldIdMap[part['name']] = part['id']
-
- self.jiras = []
- at=0
- end=1
- count=100
- while (at < end):
- params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join(versions)+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
- resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
- data = json.loads(resp.read())
- if (data.has_key('errorMessages')):
- raise Exception(data['errorMessages'])
- at = data['startAt'] + data['maxResults']
- end = data['total']
- self.jiras.extend(data['issues'])
-
- self.iter = self.jiras.__iter__()
-
- def __iter__(self):
- return self
-
- def next(self):
- data = self.iter.next()
- j = Jira(data, self)
- return j
-
-class Outputs:
- """Several different files to output to at the same time"""
-
- def __init__(self, base_file_name, file_name_pattern, keys, params={}):
- self.params = params
- self.base = open(base_file_name%params, 'w')
- self.others = {}
- for key in keys:
- both = dict(params)
- both['key'] = key
- self.others[key] = open(file_name_pattern%both, 'w')
-
- def writeAll(self, pattern):
- both = dict(self.params)
- both['key'] = ''
- self.base.write(pattern%both)
- for key in self.others.keys():
- both = dict(self.params)
- both['key'] = key
- self.others[key].write(pattern%both)
-
- def writeKeyRaw(self, key, str):
- self.base.write(str)
- if (self.others.has_key(key)):
- self.others[key].write(str)
-
- def close(self):
- self.base.close()
- for fd in self.others.values():
- fd.close()
-
-def main():
- parser = OptionParser(usage="usage: %prog [options] [USER-ignored] [PASSWORD-ignored] [VERSION]")
- parser.add_option("-v", "--version", dest="versions",
- action="append", type="string",
- help="versions in JIRA to include in releasenotes", metavar="VERSION")
- parser.add_option("--previousVer", dest="previousVer",
- action="store", type="string",
- help="previous version to include in releasenotes", metavar="VERSION")
-
- (options, args) = parser.parse_args()
-
- if (options.versions == None):
- options.versions = []
-
- if (len(args) > 2):
- options.versions.append(args[2])
-
- if (len(options.versions) <= 0):
- parser.error("At least one version needs to be supplied")
-
- versions = [ Version(v) for v in options.versions];
- versions.sort();
-
- maxVersion = str(versions[-1])
- if(options.previousVer == None):
- options.previousVer = str(versions[0].decBugFix())
- print >> sys.stderr, "WARNING: no previousVersion given, guessing it is "+options.previousVer
-
- list = JiraIter(options.versions)
- version = maxVersion
- outputs = Outputs("releasenotes.%(ver)s.html",
- "releasenotes.%(key)s.%(ver)s.html",
- ["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "previousVer":options.previousVer})
-
- head = '<META http-equiv="Content-Type" content="text/html; charset=UTF-8">\n' \
- '<title>Hadoop %(key)s %(ver)s Release Notes</title>\n' \
- '<STYLE type="text/css">\n' \
- ' H1 {font-family: sans-serif}\n' \
- ' H2 {font-family: sans-serif; margin-left: 7mm}\n' \
- ' TABLE {margin-left: 7mm}\n' \
- '</STYLE>\n' \
- '</head>\n' \
- '<body>\n' \
- '<h1>Hadoop %(key)s %(ver)s Release Notes</h1>\n' \
- 'These release notes include new developer and user-facing incompatibilities, features, and major improvements. \n' \
- '<a name="changes"/>\n' \
- '<h2>Changes since Hadoop %(previousVer)s</h2>\n' \
- '<ul>\n'
-
- outputs.writeAll(head)
-
- for jira in list:
- line = '<li> <a href="https://issues.apache.org/jira/browse/%s">%s</a>.\n' \
- ' %s %s reported by %s and fixed by %s %s<br>\n' \
- ' <b>%s</b><br>\n' \
- ' <blockquote>%s</blockquote></li>\n' \
- % (quoteHtml(jira.getId()), quoteHtml(jira.getId()), clean(jira.getPriority()), clean(jira.getType()).lower(),
- quoteHtml(jira.getReporter()), quoteHtml(jira.getAssignee()), formatComponents(jira.getComponents()),
- quoteHtml(jira.getSummary()), quoteHtml(jira.getReleaseNote()))
- outputs.writeKeyRaw(jira.getProject(), line)
-
- outputs.writeAll("</ul>\n</body></html>\n")
- outputs.close()
-
-if __name__ == "__main__":
- main()
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index fa98a0c..c92e378 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -21,6 +21,8 @@ Trunk (Unreleased)
HADOOP-11553. Formalize the shell API (aw)
+ HADOOP-11731. Rework the changelog and releasenotes (aw)
+
NEW FEATURES
HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via aw)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 706f5b5..9ca2c9b 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -479,6 +479,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
@@ -901,6 +902,56 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>releasedocs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>releasedocs</id>
+ <phase>pre-site</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>python</executable>
+ <workingDirectory>src/site/markdown/release/</workingDirectory>
+ <requiresOnline>true</requiresOnline>
+ <arguments>
+ <argument>${basedir}/../../dev-support/releasedocmaker.py</argument>
+ <argument>--version</argument>
+ <argument>${project.version}</argument>
+ <argument>--index</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>src/site/markdown/release</directory>
+ <includes>
+ <include>${project.version}</include>
+ </includes>
+ <followSymlinks>false</followSymlinks>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e6b61eb/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 2e098ef..ee6b8b5 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -159,13 +159,9 @@
</menu>
<menu name="Reference" inherit="top">
- <item name="Release Notes" href="hadoop-project-dist/hadoop-common/releasenotes.html"/>
+ <item name="Changelog and Release Notes" href="hadoop-project-dist/hadoop-common/release/index.html"/>
<item name="Java API docs" href="api/index.html"/>
<item name="Unix Shell API" href="hadoop-project-dist/hadoop-common/UnixShellAPI.html"/>
- <item name="Common CHANGES.txt" href="hadoop-project-dist/hadoop-common/CHANGES.txt"/>
- <item name="HDFS CHANGES.txt" href="hadoop-project-dist/hadoop-hdfs/CHANGES.txt"/>
- <item name="MapReduce CHANGES.txt" href="hadoop-project-dist/hadoop-mapreduce/CHANGES.txt"/>
- <item name="YARN CHANGES.txt" href="hadoop-project-dist/hadoop-yarn/CHANGES.txt"/>
<item name="Metrics" href="hadoop-project-dist/hadoop-common/Metrics.html"/>
</menu>
[38/50] [abbrv] hadoop git commit: HDFS-7996. After swapping a volume,
BlockReceiver reports ReplicaNotFoundException (Lei (Eddy) Xu via Colin P.
McCabe)
Posted by zj...@apache.org.
HDFS-7996. After swapping a volume, BlockReceiver reports ReplicaNotFoundException (Lei (Eddy) Xu via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5864e886
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5864e886
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5864e886
Branch: refs/heads/YARN-2928
Commit: 5864e88653b444d44a808003ee5668d71d5be020
Parents: fb13303
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Apr 3 14:19:46 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:16 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/datanode/BlockReceiver.java | 51 ++++++++++++--------
.../datanode/TestDataNodeHotSwapVolumes.java | 30 +++++++++++-
3 files changed, 64 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5864e886/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ae56f1b..2d399a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1373,6 +1373,9 @@ Release 2.7.0 - UNRELEASED
HDFS-8039. Fix TestDebugAdmin#testRecoverLease and
testVerifyBlockChecksumCommand on Windows. (Xiaoyu Yao via cnauroth)
+ HDFS-7996. After swapping a volume, BlockReceiver reports
+ ReplicaNotFoundException (Lei (Eddy) Xu via Colin P. McCabe)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5864e886/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 4e8ce94..58cb8b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -281,7 +281,7 @@ class BlockReceiver implements Closeable {
}
/**
- * close files.
+ * close files and release volume reference.
*/
@Override
public void close() throws IOException {
@@ -798,17 +798,20 @@ class BlockReceiver implements Closeable {
// then finalize block or convert temporary to RBW.
// For client-writes, the block is finalized in the PacketResponder.
if (isDatanode || isTransfer) {
- // close the block/crc files
- close();
- block.setNumBytes(replicaInfo.getNumBytes());
-
- if (stage == BlockConstructionStage.TRANSFER_RBW) {
- // for TRANSFER_RBW, convert temporary to RBW
- datanode.data.convertTemporaryToRbw(block);
- } else {
- // for isDatnode or TRANSFER_FINALIZED
- // Finalize the block.
- datanode.data.finalizeBlock(block);
+ // Hold a volume reference to finalize block.
+ try (ReplicaHandler handler = claimReplicaHandler()) {
+ // close the block/crc files
+ close();
+ block.setNumBytes(replicaInfo.getNumBytes());
+
+ if (stage == BlockConstructionStage.TRANSFER_RBW) {
+ // for TRANSFER_RBW, convert temporary to RBW
+ datanode.data.convertTemporaryToRbw(block);
+ } else {
+ // for isDatnode or TRANSFER_FINALIZED
+ // Finalize the block.
+ datanode.data.finalizeBlock(block);
+ }
}
datanode.metrics.incrBlocksWritten();
}
@@ -980,7 +983,14 @@ class BlockReceiver implements Closeable {
}
return partialCrc;
}
-
+
+ /** The caller claims the ownership of the replica handler. */
+ private ReplicaHandler claimReplicaHandler() {
+ ReplicaHandler handler = replicaHandler;
+ replicaHandler = null;
+ return handler;
+ }
+
private static enum PacketResponderType {
NON_PIPELINE, LAST_IN_PIPELINE, HAS_DOWNSTREAM_IN_PIPELINE
}
@@ -1280,12 +1290,15 @@ class BlockReceiver implements Closeable {
* @param startTime time when BlockReceiver started receiving the block
*/
private void finalizeBlock(long startTime) throws IOException {
- BlockReceiver.this.close();
- final long endTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime()
- : 0;
- block.setNumBytes(replicaInfo.getNumBytes());
- datanode.data.finalizeBlock(block);
-
+ long endTime = 0;
+ // Hold a volume reference to finalize block.
+ try (ReplicaHandler handler = BlockReceiver.this.claimReplicaHandler()) {
+ BlockReceiver.this.close();
+ endTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0;
+ block.setNumBytes(replicaInfo.getNumBytes());
+ datanode.data.finalizeBlock(block);
+ }
+
if (pinning) {
datanode.data.setPinning(block);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5864e886/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index f5772e3..668084b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -64,6 +65,8 @@ import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.hamcrest.CoreMatchers.anyOf;
@@ -77,6 +80,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.timeout;
public class TestDataNodeHotSwapVolumes {
@@ -577,6 +581,7 @@ public class TestDataNodeHotSwapVolumes {
final DataNode dn = cluster.getDataNodes().get(dataNodeIdx);
final FileSystem fs = cluster.getFileSystem();
final Path testFile = new Path("/test");
+ final long lastTimeDiskErrorCheck = dn.getLastDiskErrorCheck();
FSDataOutputStream out = fs.create(testFile, REPLICATION);
@@ -586,6 +591,23 @@ public class TestDataNodeHotSwapVolumes {
out.write(writeBuf);
out.hflush();
+ // Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
+ // BlockReceiver releases volume reference before finalizeBlock(), the blocks
+ // on the volume will be removed, and finalizeBlock() throws IOE.
+ final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
+ dn.data = Mockito.spy(data);
+ doAnswer(new Answer<Object>() {
+ public Object answer(InvocationOnMock invocation)
+ throws IOException, InterruptedException {
+ Thread.sleep(1000);
+ // Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
+ // the block is not removed, since the volume reference should not
+ // be released at this point.
+ data.finalizeBlock((ExtendedBlock) invocation.getArguments()[0]);
+ return null;
+ }
+ }).when(dn.data).finalizeBlock(any(ExtendedBlock.class));
+
final CyclicBarrier barrier = new CyclicBarrier(2);
List<String> oldDirs = getDataDirs(dn);
@@ -612,13 +634,19 @@ public class TestDataNodeHotSwapVolumes {
out.hflush();
out.close();
+ reconfigThread.join();
+
// Verify the file has sufficient replications.
DFSTestUtil.waitReplication(fs, testFile, REPLICATION);
// Read the content back
byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
assertEquals(BLOCK_SIZE, content.length);
- reconfigThread.join();
+ // If an IOException thrown from BlockReceiver#run, it triggers
+ // DataNode#checkDiskError(). So we can test whether checkDiskError() is called,
+ // to see whether there is IOException in BlockReceiver#run().
+ assertEquals(lastTimeDiskErrorCheck, dn.getLastDiskErrorCheck());
+
if (!exceptions.isEmpty()) {
throw new IOException(exceptions.get(0).getCause());
}
[21/50] [abbrv] hadoop git commit: HADOOP-11757. NFS gateway should
shutdown when it can't start UDP or TCP server. Contributed by Brandon Li
Posted by zj...@apache.org.
HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP server. Contributed by Brandon Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09686762
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09686762
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09686762
Branch: refs/heads/YARN-2928
Commit: 096867629333d4b2de4e7b971b758fbabc11d1d3
Parents: f6d4143
Author: Brandon Li <br...@apache.org>
Authored: Wed Apr 1 17:04:44 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 ++-
.../org/apache/hadoop/mount/MountdBase.java | 26 +++++++++++++--
.../org/apache/hadoop/nfs/nfs3/Nfs3Base.java | 15 +++++++--
.../apache/hadoop/oncrpc/SimpleTcpServer.java | 31 ++++++++++++------
.../apache/hadoop/oncrpc/SimpleUdpServer.java | 33 +++++++++++++-------
5 files changed, 82 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 111fb5e..fa98a0c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1186,7 +1186,10 @@ Release 2.7.0 - UNRELEASED
HADOOP-11787. OpensslSecureRandom.c pthread_threadid_np usage signature is
wrong on 32-bit Mac. (Kiran Kumar M R via cnauroth)
-
+
+ HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP
+ server (brandonli)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java
index 8d7d6dc..92ca7ec 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountdBase.java
@@ -60,7 +60,17 @@ abstract public class MountdBase {
SimpleUdpServer udpServer = new SimpleUdpServer(rpcProgram.getPort(),
rpcProgram, 1);
rpcProgram.startDaemons();
- udpServer.run();
+ try {
+ udpServer.run();
+ } catch (Throwable e) {
+ LOG.fatal("Failed to start the UDP server.", e);
+ if (udpServer.getBoundPort() > 0) {
+ rpcProgram.unregister(PortmapMapping.TRANSPORT_UDP,
+ udpServer.getBoundPort());
+ }
+ udpServer.shutdown();
+ terminate(1, e);
+ }
udpBoundPort = udpServer.getBoundPort();
}
@@ -69,7 +79,17 @@ abstract public class MountdBase {
SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.getPort(),
rpcProgram, 1);
rpcProgram.startDaemons();
- tcpServer.run();
+ try {
+ tcpServer.run();
+ } catch (Throwable e) {
+ LOG.fatal("Failed to start the TCP server.", e);
+ if (tcpServer.getBoundPort() > 0) {
+ rpcProgram.unregister(PortmapMapping.TRANSPORT_TCP,
+ tcpServer.getBoundPort());
+ }
+ tcpServer.shutdown();
+ terminate(1, e);
+ }
tcpBoundPort = tcpServer.getBoundPort();
}
@@ -83,7 +103,7 @@ abstract public class MountdBase {
rpcProgram.register(PortmapMapping.TRANSPORT_UDP, udpBoundPort);
rpcProgram.register(PortmapMapping.TRANSPORT_TCP, tcpBoundPort);
} catch (Throwable e) {
- LOG.fatal("Failed to start the server. Cause:", e);
+ LOG.fatal("Failed to register the MOUNT service.", e);
terminate(1, e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
index 40744bc..80faca5 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Base.java
@@ -29,7 +29,6 @@ import static org.apache.hadoop.util.ExitUtil.terminate;
/**
* Nfs server. Supports NFS v3 using {@link RpcProgram}.
- * Currently Mountd program is also started inside this class.
* Only TCP server is supported and UDP is not supported.
*/
public abstract class Nfs3Base {
@@ -55,7 +54,7 @@ public abstract class Nfs3Base {
try {
rpcProgram.register(PortmapMapping.TRANSPORT_TCP, nfsBoundPort);
} catch (Throwable e) {
- LOG.fatal("Failed to start the server. Cause:", e);
+ LOG.fatal("Failed to register the NFSv3 service.", e);
terminate(1, e);
}
}
@@ -65,7 +64,17 @@ public abstract class Nfs3Base {
SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.getPort(),
rpcProgram, 0);
rpcProgram.startDaemons();
- tcpServer.run();
+ try {
+ tcpServer.run();
+ } catch (Throwable e) {
+ LOG.fatal("Failed to start the TCP server.", e);
+ if (tcpServer.getBoundPort() > 0) {
+ rpcProgram.unregister(PortmapMapping.TRANSPORT_TCP,
+ tcpServer.getBoundPort());
+ }
+ tcpServer.shutdown();
+ terminate(1, e);
+ }
nfsBoundPort = tcpServer.getBoundPort();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
index 949fdca..99d1d6f 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java
@@ -39,7 +39,9 @@ public class SimpleTcpServer {
protected final int port;
protected int boundPort = -1; // Will be set after server starts
protected final SimpleChannelUpstreamHandler rpcProgram;
-
+ private ServerBootstrap server;
+ private Channel ch;
+
/** The maximum number of I/O worker threads */
protected final int workerCount;
@@ -53,7 +55,7 @@ public class SimpleTcpServer {
this.rpcProgram = program;
this.workerCount = workercount;
}
-
+
public void run() {
// Configure the Server.
ChannelFactory factory;
@@ -66,9 +68,9 @@ public class SimpleTcpServer {
Executors.newCachedThreadPool(), Executors.newCachedThreadPool(),
workerCount);
}
-
- ServerBootstrap bootstrap = new ServerBootstrap(factory);
- bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
+
+ server = new ServerBootstrap(factory);
+ server.setPipelineFactory(new ChannelPipelineFactory() {
@Override
public ChannelPipeline getPipeline() throws Exception {
@@ -77,14 +79,14 @@ public class SimpleTcpServer {
RpcUtil.STAGE_RPC_TCP_RESPONSE);
}
});
- bootstrap.setOption("child.tcpNoDelay", true);
- bootstrap.setOption("child.keepAlive", true);
-
+ server.setOption("child.tcpNoDelay", true);
+ server.setOption("child.keepAlive", true);
+
// Listen to TCP port
- Channel ch = bootstrap.bind(new InetSocketAddress(port));
+ ch = server.bind(new InetSocketAddress(port));
InetSocketAddress socketAddr = (InetSocketAddress) ch.getLocalAddress();
boundPort = socketAddr.getPort();
-
+
LOG.info("Started listening to TCP requests at port " + boundPort + " for "
+ rpcProgram + " with workerCount " + workerCount);
}
@@ -93,4 +95,13 @@ public class SimpleTcpServer {
public int getBoundPort() {
return this.boundPort;
}
+
+ public void shutdown() {
+ if (ch != null) {
+ ch.close().awaitUninterruptibly();
+ }
+ if (server != null) {
+ server.releaseExternalResources();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09686762/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
index 8e77fce..c9cb2b3 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleUdpServer.java
@@ -41,8 +41,11 @@ public class SimpleUdpServer {
protected final SimpleChannelUpstreamHandler rpcProgram;
protected final int workerCount;
protected int boundPort = -1; // Will be set after server starts
+ private ConnectionlessBootstrap server;
+ private Channel ch;
- public SimpleUdpServer(int port, SimpleChannelUpstreamHandler program, int workerCount) {
+ public SimpleUdpServer(int port, SimpleChannelUpstreamHandler program,
+ int workerCount) {
this.port = port;
this.rpcProgram = program;
this.workerCount = workerCount;
@@ -53,20 +56,19 @@ public class SimpleUdpServer {
DatagramChannelFactory f = new NioDatagramChannelFactory(
Executors.newCachedThreadPool(), workerCount);
- ConnectionlessBootstrap b = new ConnectionlessBootstrap(f);
- b.setPipeline(Channels.pipeline(
- RpcUtil.STAGE_RPC_MESSAGE_PARSER, rpcProgram,
- RpcUtil.STAGE_RPC_UDP_RESPONSE));
+ server = new ConnectionlessBootstrap(f);
+ server.setPipeline(Channels.pipeline(RpcUtil.STAGE_RPC_MESSAGE_PARSER,
+ rpcProgram, RpcUtil.STAGE_RPC_UDP_RESPONSE));
+
+ server.setOption("broadcast", "false");
+ server.setOption("sendBufferSize", SEND_BUFFER_SIZE);
+ server.setOption("receiveBufferSize", RECEIVE_BUFFER_SIZE);
- b.setOption("broadcast", "false");
- b.setOption("sendBufferSize", SEND_BUFFER_SIZE);
- b.setOption("receiveBufferSize", RECEIVE_BUFFER_SIZE);
-
// Listen to the UDP port
- Channel ch = b.bind(new InetSocketAddress(port));
+ ch = server.bind(new InetSocketAddress(port));
InetSocketAddress socketAddr = (InetSocketAddress) ch.getLocalAddress();
boundPort = socketAddr.getPort();
-
+
LOG.info("Started listening to UDP requests at port " + boundPort + " for "
+ rpcProgram + " with workerCount " + workerCount);
}
@@ -75,4 +77,13 @@ public class SimpleUdpServer {
public int getBoundPort() {
return this.boundPort;
}
+
+ public void shutdown() {
+ if (ch != null) {
+ ch.close().awaitUninterruptibly();
+ }
+ if (server != null) {
+ server.releaseExternalResources();
+ }
+ }
}
[32/50] [abbrv] hadoop git commit: HDFS-7893. Update the POM to
create a separate hdfs-client jar. Contributed by Haohui Mai.
Posted by zj...@apache.org.
HDFS-7893. Update the POM to create a separate hdfs-client jar. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88c14681
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88c14681
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88c14681
Branch: refs/heads/YARN-2928
Commit: 88c146814b3431a8eb98c03a0ed706a675058b4e
Parents: e75307b
Author: Haohui Mai <wh...@apache.org>
Authored: Thu Apr 2 15:29:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:14 2015 -0700
----------------------------------------------------------------------
.../dev-support/findbugsExcludeFile.xml | 2 +
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 49 ++++++++++++++++++++
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 ++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 ++
.../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 5 ++
hadoop-hdfs-project/pom.xml | 1 +
hadoop-project/pom.xml | 5 ++
7 files changed, 72 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
new file mode 100644
index 0000000..43bc332
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
@@ -0,0 +1,2 @@
+<FindBugsFilter>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
new file mode 100644
index 0000000..900f345
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project-dist</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>../../hadoop-project-dist</relativePath>
+ </parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ <description>Apache Hadoop HDFS Client</description>
+ <name>Apache Hadoop HDFS Client</name>
+ <packaging>jar</packaging>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>dev-support/findbugsExcludeFile.xml</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 9a9d29c..ac8930c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -58,6 +58,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 2d402a9..c11b963 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -61,6 +61,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<type>test-jar</type>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
index 6df6d98..3b205e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
@@ -52,6 +52,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 920bb4b..0a61c46 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -32,6 +32,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modules>
<module>hadoop-hdfs</module>
+ <module>hadoop-hdfs-client</module>
<module>hadoop-hdfs-httpfs</module>
<module>hadoop-hdfs/src/contrib/bkjournal</module>
<module>hadoop-hdfs-nfs</module>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88c14681/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3ccc332..2444c23 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -160,6 +160,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
[19/50] [abbrv] hadoop git commit: HDFS-8026. Trace
FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk
(cmccabe)
Posted by zj...@apache.org.
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4afce08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4afce08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4afce08
Branch: refs/heads/YARN-2928
Commit: f4afce0801bc488e301aee820a8091252bbc2748
Parents: f066e73
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Apr 1 13:55:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:12 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/FSOutputSummer.java | 20 ++++++++++++++++----
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++-----------
.../org/apache/hadoop/tracing/TestTracing.java | 4 ++--
4 files changed, 25 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index 13a5e26..d2998b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.NullScope;
+import org.apache.htrace.TraceScope;
import java.io.IOException;
import java.io.OutputStream;
@@ -194,16 +196,26 @@ abstract public class FSOutputSummer extends OutputStream {
return sum.getChecksumSize();
}
+ protected TraceScope createWriteTraceScope() {
+ return NullScope.INSTANCE;
+ }
+
/** Generate checksums for the given data chunks and output chunks & checksums
* to the underlying output stream.
*/
private void writeChecksumChunks(byte b[], int off, int len)
throws IOException {
sum.calculateChunkedSums(b, off, len, checksum, 0);
- for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
- int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
- int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
- writeChunk(b, off + i, chunkLen, checksum, ckOffset, getChecksumSize());
+ TraceScope scope = createWriteTraceScope();
+ try {
+ for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
+ int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
+ int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
+ writeChunk(b, off + i, chunkLen, checksum, ckOffset,
+ getChecksumSize());
+ }
+ } finally {
+ scope.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 435fdd7..b5591e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -378,6 +378,9 @@ Release 2.8.0 - UNRELEASED
OPTIMIZATIONS
+ HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
+ DFSOutputStream#writeChunk (cmccabe)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 933d8e6..c88639d 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -372,21 +372,14 @@ public class DFSOutputStream extends FSOutputSummer
}
}
+ protected TraceScope createWriteTraceScope() {
+ return dfsClient.getPathTraceScope("DFSOutputStream#write", src);
+ }
+
// @see FSOutputSummer#writeChunk()
@Override
protected synchronized void writeChunk(byte[] b, int offset, int len,
byte[] checksum, int ckoff, int cklen) throws IOException {
- TraceScope scope =
- dfsClient.getPathTraceScope("DFSOutputStream#writeChunk", src);
- try {
- writeChunkImpl(b, offset, len, checksum, ckoff, cklen);
- } finally {
- scope.close();
- }
- }
-
- private synchronized void writeChunkImpl(byte[] b, int offset, int len,
- byte[] checksum, int ckoff, int cklen) throws IOException {
dfsClient.checkOpen();
checkClosed();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4afce08/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 3720abe..01361b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -89,7 +89,7 @@ public class TestTracing {
"org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
"ClientNamenodeProtocol#complete",
"newStreamForCreate",
- "DFSOutputStream#writeChunk",
+ "DFSOutputStream#write",
"DFSOutputStream#close",
"dataStreamer",
"OpWriteBlockProto",
@@ -117,7 +117,7 @@ public class TestTracing {
"org.apache.hadoop.hdfs.protocol.ClientProtocol.complete",
"ClientNamenodeProtocol#complete",
"newStreamForCreate",
- "DFSOutputStream#writeChunk",
+ "DFSOutputStream#write",
"DFSOutputStream#close",
};
for (String desc : spansInTopTrace) {
[11/50] [abbrv] hadoop git commit: YARN-3248. Display count of nodes
blacklisted by apps in the web UI. Contributed by Varun Vasudev
Posted by zj...@apache.org.
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
Contributed by Varun Vasudev
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40f362b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40f362b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40f362b1
Branch: refs/heads/YARN-2928
Commit: 40f362b1cbbae7d86d478e781d099e4de5b6ac92
Parents: db2998e
Author: Xuan <xg...@apache.org>
Authored: Wed Apr 1 04:19:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:10 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/server/webapp/AppAttemptBlock.java | 66 +++++----
.../hadoop/yarn/server/webapp/AppBlock.java | 37 +++--
.../hadoop/yarn/server/webapp/AppsBlock.java | 85 ++++++-----
.../scheduler/AppSchedulingInfo.java | 4 +
.../scheduler/SchedulerApplicationAttempt.java | 4 +
.../webapp/AppsBlockWithMetrics.java | 2 +-
.../webapp/CapacitySchedulerPage.java | 2 +-
.../webapp/RMAppAttemptBlock.java | 67 +++++++++
.../resourcemanager/webapp/RMAppBlock.java | 110 ++++++++++++++
.../resourcemanager/webapp/RMAppsBlock.java | 146 +++++++++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 3 +-
.../webapp/dao/AppAttemptInfo.java | 19 ++-
.../webapp/TestRMWebServicesApps.java | 2 +-
14 files changed, 468 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f18cf30..461f07d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -920,6 +920,9 @@ Release 2.7.0 - UNRELEASED
removing inconsistencies in the default values. (Junping Du and Karthik
Kambatla via vinodkv)
+ YARN-3248. Display count of nodes blacklisted by apps in the web UI.
+ (Varun Vasudev via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index dca39d6..8df94e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -72,7 +72,7 @@ public class AppAttemptBlock extends HtmlBlock {
}
UserGroupInformation callerUGI = getCallerUGI();
- ApplicationAttemptReport appAttemptReport = null;
+ ApplicationAttemptReport appAttemptReport;
try {
final GetApplicationAttemptReportRequest request =
GetApplicationAttemptReportRequest.newInstance(appAttemptId);
@@ -135,34 +135,7 @@ public class AppAttemptBlock extends HtmlBlock {
&& appAttempt.getRpcPort() < 65536) {
node = appAttempt.getHost() + ":" + appAttempt.getRpcPort();
}
- info("Application Attempt Overview")
- ._(
- "Application Attempt State:",
- appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
- .getAppAttemptState())
- ._(
- "AM Container:",
- appAttempt.getAmContainerId() == null || containers == null
- || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
- ? null : root_url("container", appAttempt.getAmContainerId()),
- String.valueOf(appAttempt.getAmContainerId()))
- ._("Node:", node)
- ._(
- "Tracking URL:",
- appAttempt.getTrackingUrl() == null
- || appAttempt.getTrackingUrl() == UNAVAILABLE ? null
- : root_url(appAttempt.getTrackingUrl()),
- appAttempt.getTrackingUrl() == null
- || appAttempt.getTrackingUrl() == UNAVAILABLE
- ? "Unassigned"
- : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
- || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
- || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
- ? "History" : "ApplicationMaster")
- ._("Diagnostics Info:", appAttempt.getDiagnosticsInfo() == null ?
- "" : appAttempt.getDiagnosticsInfo());
-
-
+ generateOverview(appAttemptReport, containers, appAttempt, node);
if (exceptionWhenGetContainerReports) {
html
@@ -218,7 +191,40 @@ public class AppAttemptBlock extends HtmlBlock {
tbody._()._();
}
- private boolean hasAMContainer(ContainerId containerId,
+ protected void generateOverview(ApplicationAttemptReport appAttemptReport,
+ Collection<ContainerReport> containers, AppAttemptInfo appAttempt,
+ String node) {
+ info("Application Attempt Overview")
+ ._(
+ "Application Attempt State:",
+ appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
+ .getAppAttemptState())
+ ._(
+ "AM Container:",
+ appAttempt.getAmContainerId() == null || containers == null
+ || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
+ ? null : root_url("container", appAttempt.getAmContainerId()),
+ String.valueOf(appAttempt.getAmContainerId()))
+ ._("Node:", node)
+ ._(
+ "Tracking URL:",
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null
+ : root_url(appAttempt.getTrackingUrl()),
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE)
+ ? "Unassigned"
+ : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
+ ? "History" : "ApplicationMaster")
+ ._(
+ "Diagnostics Info:",
+ appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt
+ .getDiagnosticsInfo());
+ }
+
+ protected boolean hasAMContainer(ContainerId containerId,
Collection<ContainerReport> containers) {
for (ContainerReport container : containers) {
if (containerId.equals(container.getContainerId())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index abb6b9c..ae4737d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -88,7 +88,7 @@ public class AppBlock extends HtmlBlock {
}
UserGroupInformation callerUGI = getCallerUGI();
- ApplicationReport appReport = null;
+ ApplicationReport appReport;
try {
final GetApplicationReportRequest request =
GetApplicationReportRequest.newInstance(appID);
@@ -160,7 +160,8 @@ public class AppBlock extends HtmlBlock {
._("Application Type:", app.getType())
._("Application Tags:",
app.getApplicationTags() == null ? "" : app.getApplicationTags())
- ._("YarnApplicationState:",
+ ._(
+ "YarnApplicationState:",
app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app
.getAppState()))
._("FinalStatus Reported by AM:",
@@ -170,16 +171,19 @@ public class AppBlock extends HtmlBlock {
"Elapsed:",
StringUtils.formatTime(Times.elapsed(app.getStartedTime(),
app.getFinishedTime())))
- ._("Tracking URL:",
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? null : root_url(app.getTrackingUrl()),
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? "Unassigned" : app.getAppState() == YarnApplicationState.FINISHED
- || app.getAppState() == YarnApplicationState.FAILED
- || app.getAppState() == YarnApplicationState.KILLED ? "History"
- : "ApplicationMaster")
+ ._(
+ "Tracking URL:",
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app
+ .getTrackingUrl()),
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
+ .getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED ? "History"
+ : "ApplicationMaster")
._("Diagnostics:",
- app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
+ app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
Collection<ApplicationAttemptReport> attempts;
try {
@@ -210,6 +214,13 @@ public class AppBlock extends HtmlBlock {
html._(InfoBlock.class);
+ generateApplicationTable(html, callerUGI, attempts);
+
+ }
+
+ protected void generateApplicationTable(Block html,
+ UserGroupInformation callerUGI,
+ Collection<ApplicationAttemptReport> attempts) {
// Application Attempt Table
TBODY<TABLE<Hamlet>> tbody =
html.table("#attempts").thead().tr().th(".id", "Attempt ID")
@@ -219,7 +230,7 @@ public class AppBlock extends HtmlBlock {
StringBuilder attemptsTableData = new StringBuilder("[\n");
for (final ApplicationAttemptReport appAttemptReport : attempts) {
AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
- ContainerReport containerReport = null;
+ ContainerReport containerReport;
try {
// AM container is always the first container of the attempt
final GetContainerReportRequest request =
@@ -230,7 +241,7 @@ public class AppBlock extends HtmlBlock {
appBaseProt.getContainerReport(request).getContainerReport();
} else {
containerReport = callerUGI.doAs(
- new PrivilegedExceptionAction<ContainerReport> () {
+ new PrivilegedExceptionAction<ContainerReport>() {
@Override
public ContainerReport run() throws Exception {
ContainerReport report = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 161486d..6a35349 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.EnumSet;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -47,26 +49,19 @@ public class AppsBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppsBlock.class);
protected ApplicationBaseProtocol appBaseProt;
+ protected EnumSet<YarnApplicationState> reqAppStates;
+ protected UserGroupInformation callerUGI;
+ protected Collection<ApplicationReport> appReports;
@Inject
- AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
+ protected AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
super(ctx);
this.appBaseProt = appBaseProt;
}
- @Override
- public void render(Block html) {
- setTitle("Applications");
-
- TBODY<TABLE<Hamlet>> tbody =
- html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
- .th(".name", "Name").th(".type", "Application Type")
- .th(".queue", "Queue").th(".starttime", "StartTime")
- .th(".finishtime", "FinishTime").th(".state", "State")
- .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
- .th(".ui", "Tracking UI")._()._().tbody();
- EnumSet<YarnApplicationState> reqAppStates =
- EnumSet.noneOf(YarnApplicationState.class);
+ protected void fetchData() throws YarnException, IOException,
+ InterruptedException {
+ reqAppStates = EnumSet.noneOf(YarnApplicationState.class);
String reqStateString = $(APP_STATE);
if (reqStateString != null && !reqStateString.isEmpty()) {
String[] appStateStrings = reqStateString.split(",");
@@ -75,28 +70,49 @@ public class AppsBlock extends HtmlBlock {
}
}
- UserGroupInformation callerUGI = getCallerUGI();
- Collection<ApplicationReport> appReports = null;
+ callerUGI = getCallerUGI();
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance(reqAppStates);
+ if (callerUGI == null) {
+ appReports = appBaseProt.getApplications(request).getApplicationList();
+ } else {
+ appReports =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<Collection<ApplicationReport>>() {
+ @Override
+ public Collection<ApplicationReport> run() throws Exception {
+ return appBaseProt.getApplications(request)
+ .getApplicationList();
+ }
+ });
+ }
+ }
+
+ @Override
+ public void render(Block html) {
+ setTitle("Applications");
+
try {
- final GetApplicationsRequest request =
- GetApplicationsRequest.newInstance(reqAppStates);
- if (callerUGI == null) {
- appReports = appBaseProt.getApplications(request).getApplicationList();
- } else {
- appReports = callerUGI.doAs(
- new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
- @Override
- public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(request).getApplicationList();
- }
- });
- }
- } catch (Exception e) {
+ fetchData();
+ }
+ catch( Exception e) {
String message = "Failed to read the applications.";
LOG.error(message, e);
html.p()._(message)._();
return;
}
+ renderData(html);
+ }
+
+ protected void renderData(Block html) {
+ TBODY<TABLE<Hamlet>> tbody =
+ html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
+ .th(".name", "Name").th(".type", "Application Type")
+ .th(".queue", "Queue").th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime").th(".state", "State")
+ .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+ .th(".ui", "Tracking UI")._()._().tbody();
+
StringBuilder appsTableData = new StringBuilder("[\n");
for (ApplicationReport appReport : appReports) {
// TODO: remove the following condition. It is still here because
@@ -117,7 +133,7 @@ public class AppsBlock extends HtmlBlock {
.append("</a>\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
- .getUser())))
+ .getUser())))
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
@@ -144,11 +160,12 @@ public class AppsBlock extends HtmlBlock {
.append("'> </div> </div>").append("\",\"<a ");
String trackingURL =
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? null : app.getTrackingUrl();
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : app
+ .getTrackingUrl();
String trackingUI =
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
+ app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE)
? "Unassigned"
: app.getAppState() == YarnApplicationState.FINISHED
|| app.getAppState() == YarnApplicationState.FAILED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 84ebe9c..5521d47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -462,6 +462,10 @@ public class AppSchedulingInfo {
return this.blacklist;
}
+ public synchronized Set<String> getBlackListCopy() {
+ return new HashSet<>(this.blacklist);
+ }
+
public synchronized void transferStateFromPreviousAppSchedulingInfo(
AppSchedulingInfo appInfo) {
// this.priorities = appInfo.getPriorities();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index bf5641d..5e0bbc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -665,4 +665,8 @@ public class SchedulerApplicationAttempt {
}
}
}
+
+ public Set<String> getBlacklistedNodes() {
+ return this.appSchedulingInfo.getBlackListCopy();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
index cb0836a..29889ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
@@ -27,6 +27,6 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
class AppsBlockWithMetrics extends HtmlBlock {
@Override public void render(Block html) {
html._(MetricsOverviewTable.class);
- html._(AppsBlock.class);
+ html._(RMAppsBlock.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index c6c410c..e62fd70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -239,7 +239,7 @@ class CapacitySchedulerPage extends RmView {
ul._()._().
script().$type("text/javascript").
_("$('#cs').hide();")._()._().
- _(AppsBlock.class);
+ _(RMAppsBlock.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 419c0ce..1861874 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -23,8 +23,12 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@@ -32,8 +36,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
@@ -43,6 +50,9 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import java.util.Collection;
+import java.util.Set;
+
public class RMAppAttemptBlock extends AppAttemptBlock{
private final ResourceManager rm;
@@ -174,4 +184,61 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
}
return attempt;
}
+
+ protected void generateOverview(ApplicationAttemptReport appAttemptReport,
+ Collection<ContainerReport> containers, AppAttemptInfo appAttempt,
+ String node) {
+
+ String blacklistedNodes = "-";
+ Set<String> nodes =
+ getBlacklistedNodes(rm, getRMAppAttempt().getAppAttemptId());
+ if (nodes != null) {
+ if (!nodes.isEmpty()) {
+ blacklistedNodes = StringUtils.join(nodes, ", ");
+ }
+ }
+
+ info("Application Attempt Overview")
+ ._(
+ "Application Attempt State:",
+ appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
+ .getAppAttemptState())
+ ._(
+ "AM Container:",
+ appAttempt.getAmContainerId() == null || containers == null
+ || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
+ ? null : root_url("container", appAttempt.getAmContainerId()),
+ String.valueOf(appAttempt.getAmContainerId()))
+ ._("Node:", node)
+ ._(
+ "Tracking URL:",
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null
+ : root_url(appAttempt.getTrackingUrl()),
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE)
+ ? "Unassigned"
+ : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
+ ? "History" : "ApplicationMaster")
+ ._(
+ "Diagnostics Info:",
+ appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt
+ .getDiagnosticsInfo())._("Blacklisted Nodes:", blacklistedNodes);
+ }
+
+ public static Set<String> getBlacklistedNodes(ResourceManager rm,
+ ApplicationAttemptId appid) {
+ if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
+ AbstractYarnScheduler ayScheduler =
+ (AbstractYarnScheduler) rm.getResourceScheduler();
+ SchedulerApplicationAttempt attempt =
+ ayScheduler.getApplicationAttempt(appid);
+ if (attempt != null) {
+ return attempt.getBlacklistedNodes();
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 64c5747..4942079 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -20,13 +20,25 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
@@ -34,10 +46,16 @@ import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+import java.util.Set;
+
public class RMAppBlock extends AppBlock{
+ private static final Log LOG = LogFactory.getLog(RMAppBlock.class);
private final ResourceManager rm;
+
@Inject
RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
super(rm.getClientRMService(), ctx, conf);
@@ -91,4 +109,96 @@ public class RMAppBlock extends AppBlock{
appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
pdiv._();
}
+
+ @Override
+ protected void generateApplicationTable(Block html,
+ UserGroupInformation callerUGI,
+ Collection<ApplicationAttemptReport> attempts) {
+ // Application Attempt Table
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody =
+ html.table("#attempts").thead().tr().th(".id", "Attempt ID")
+ .th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
+ .th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody();
+
+ StringBuilder attemptsTableData = new StringBuilder("[\n");
+ for (final ApplicationAttemptReport appAttemptReport : attempts) {
+ AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
+ ContainerReport containerReport = null;
+ try {
+ // AM container is always the first container of the attempt
+ final GetContainerReportRequest request =
+ GetContainerReportRequest.newInstance(ContainerId.newContainerId(
+ appAttemptReport.getApplicationAttemptId(), 1));
+ if (callerUGI == null) {
+ containerReport =
+ appBaseProt.getContainerReport(request).getContainerReport();
+ } else {
+ containerReport = callerUGI.doAs(
+ new PrivilegedExceptionAction<ContainerReport>() {
+ @Override
+ public ContainerReport run() throws Exception {
+ ContainerReport report = null;
+ try {
+ report = appBaseProt.getContainerReport(request)
+ .getContainerReport();
+ } catch (ContainerNotFoundException ex) {
+ LOG.warn(ex.getMessage());
+ }
+ return report;
+ }
+ });
+ }
+ } catch (Exception e) {
+ String message =
+ "Failed to read the AM container of the application attempt "
+ + appAttemptReport.getApplicationAttemptId() + ".";
+ LOG.error(message, e);
+ html.p()._(message)._();
+ return;
+ }
+ long startTime = 0L;
+ String logsLink = null;
+ String nodeLink = null;
+ if (containerReport != null) {
+ ContainerInfo container = new ContainerInfo(containerReport);
+ startTime = container.getStartedTime();
+ logsLink = containerReport.getLogUrl();
+ nodeLink = containerReport.getNodeHttpAddress();
+ }
+ String blacklistedNodesCount = "N/A";
+ Set<String> nodes = RMAppAttemptBlock.getBlacklistedNodes(rm,
+ ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId()));
+ if(nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
+
+ // AppAttemptID numerical value parsed by parseHadoopID in
+ // yarn.dt.plugins.js
+ attemptsTableData
+ .append("[\"<a href='")
+ .append(url("appattempt", appAttempt.getAppAttemptId()))
+ .append("'>")
+ .append(appAttempt.getAppAttemptId())
+ .append("</a>\",\"")
+ .append(startTime)
+ .append("\",\"<a ")
+ .append(nodeLink == null ? "#" : "href='" + nodeLink)
+ .append("'>")
+ .append(nodeLink == null ? "N/A" : StringEscapeUtils
+ .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
+ .append("</a>\",\"<a ")
+ .append(logsLink == null ? "#" : "href='" + logsLink).append("'>")
+ .append(logsLink == null ? "N/A" : "Logs").append("</a>\",").append(
+ "\"").append(blacklistedNodesCount).append("\"],\n");
+ }
+ if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
+ attemptsTableData.delete(attemptsTableData.length() - 2,
+ attemptsTableData.length() - 1);
+ }
+ attemptsTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var attemptsTableData=" + attemptsTableData)._();
+
+ tbody._()._();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
new file mode 100644
index 0000000..94f3afa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.webapp.AppsBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.View;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+
+import com.google.inject.Inject;
+
+public class RMAppsBlock extends AppsBlock {
+
+ private ResourceManager rm;
+
+ @Inject
+ RMAppsBlock(ResourceManager rm, ApplicationBaseProtocol appBaseProt,
+ View.ViewContext ctx) {
+ super(appBaseProt, ctx);
+ this.rm = rm;
+ }
+
+ @Override
+ protected void renderData(Block html) {
+ TBODY<TABLE<Hamlet>> tbody =
+ html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
+ .th(".name", "Name").th(".type", "Application Type")
+ .th(".queue", "Queue").th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime").th(".state", "State")
+ .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+ .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()
+ ._().tbody();
+
+ StringBuilder appsTableData = new StringBuilder("[\n");
+ for (ApplicationReport appReport : appReports) {
+ // TODO: remove the following condition. It is still here because
+ // the history side implementation of ApplicationBaseProtocol
+ // hasn't filtering capability (YARN-1819).
+ if (!reqAppStates.isEmpty()
+ && !reqAppStates.contains(appReport.getYarnApplicationState())) {
+ continue;
+ }
+
+ AppInfo app = new AppInfo(appReport);
+ String blacklistedNodesCount = "N/A";
+ Set<String> nodes =
+ RMAppAttemptBlock
+ .getBlacklistedNodes(rm, ConverterUtils.toApplicationAttemptId(app
+ .getCurrentAppAttemptId()));
+ if (nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
+ String percent = String.format("%.1f", app.getProgress());
+ // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+ appsTableData
+ .append("[\"<a href='")
+ .append(url("app", app.getAppId()))
+ .append("'>")
+ .append(app.getAppId())
+ .append("</a>\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getUser())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getName())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getType())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getQueue()))).append("\",\"").append(app.getStartedTime())
+ .append("\",\"").append(app.getFinishedTime())
+ .append("\",\"")
+ .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())
+ .append("\",\"")
+ .append(app.getFinalAppStatus())
+ .append("\",\"")
+ // Progress bar
+ .append("<br title='").append(percent).append("'> <div class='")
+ .append(C_PROGRESSBAR).append("' title='").append(join(percent, '%'))
+ .append("'> ").append("<div class='").append(C_PROGRESSBAR_VALUE)
+ .append("' style='").append(join("width:", percent, '%'))
+ .append("'> </div> </div>").append("\",\"<a ");
+
+ String trackingURL =
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : app
+ .getTrackingUrl();
+
+ String trackingUI =
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
+ .getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED ? "History"
+ : "ApplicationMaster";
+ appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
+ .append("'>").append(trackingUI).append("</a>\",").append("\"")
+ .append(blacklistedNodesCount).append("\"],\n");
+
+ }
+ if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
+ appsTableData.delete(appsTableData.length() - 2,
+ appsTableData.length() - 1);
+ }
+ appsTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var appsTableData=" + appsTableData)._();
+
+ tbody._()._();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 059ea09..36f2b1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -647,7 +647,8 @@ public class RMWebServices {
AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
for (RMAppAttempt attempt : app.getAppAttempts().values()) {
- AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
+ AppAttemptInfo attemptInfo =
+ new AppAttemptInfo(rm, attempt, app.getUser());
appAttemptsInfo.add(attemptInfo);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
index 0bb9cf7..2c7bbd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
@@ -21,8 +21,13 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMAppAttemptBlock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -36,16 +41,18 @@ public class AppAttemptInfo {
protected String nodeHttpAddress;
protected String nodeId;
protected String logsLink;
+ protected String blacklistedNodes;
public AppAttemptInfo() {
}
- public AppAttemptInfo(RMAppAttempt attempt, String user) {
+ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user) {
this.startTime = 0;
this.containerId = "";
this.nodeHttpAddress = "";
this.nodeId = "";
this.logsLink = "";
+ this.blacklistedNodes = "";
if (attempt != null) {
this.id = attempt.getAppAttemptId().getAttemptId();
this.startTime = attempt.getStartTime();
@@ -57,6 +64,16 @@ public class AppAttemptInfo {
this.logsLink =
WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
ConverterUtils.toString(masterContainer.getId()), user);
+ if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
+ AbstractYarnScheduler ayScheduler =
+ (AbstractYarnScheduler) rm.getResourceScheduler();
+ SchedulerApplicationAttempt sattempt =
+ ayScheduler.getApplicationAttempt(attempt.getAppAttemptId());
+ if (sattempt != null) {
+ blacklistedNodes =
+ StringUtils.join(sattempt.getBlacklistedNodes(), ", ");
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40f362b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index c60a584..bd43c55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1598,7 +1598,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
String user)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 6, info.length());
+ assertEquals("incorrect number of elements", 7, info.length());
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
info.getLong("startTime"), info.getString("containerId"),
[12/50] [abbrv] hadoop git commit: YARN-3430. Made headroom data
available on app attempt page of RM WebUI. Contributed by Xuan Gong.
Posted by zj...@apache.org.
YARN-3430. Made headroom data available on app attempt page of RM WebUI. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f066e734
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f066e734
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f066e734
Branch: refs/heads/YARN-2928
Commit: f066e73492e9c10c747cfcc54286ac3fd9df3f1e
Parents: b9fbc73
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Apr 1 13:47:54 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:11 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f066e734/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6fbc26e..bc8d06a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -926,6 +926,9 @@ Release 2.7.0 - UNRELEASED
removing inconsistencies in the default values. (Junping Du and Karthik
Kambatla via vinodkv)
+ YARN-3430. Made headroom data available on app attempt page of RM WebUI.
+ (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f066e734/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 1861874..1831920 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -166,10 +166,12 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
if (attempt != null) {
if (!isApplicationInFinalState(YarnApplicationAttemptState
.valueOf(attempt.getAppAttemptState().toString()))) {
+ RMAppAttemptMetrics metrics = attempt.getRMAppAttemptMetrics();
DIV<Hamlet> pdiv = html._(InfoBlock.class).div(_INFO_WRAP);
info("Application Attempt Overview").clear();
info("Application Attempt Metrics")._(
- "Application Attempt Headroom : ", 0);
+ "Application Attempt Headroom : ", metrics == null ? "N/A" :
+ metrics.getApplicationAttemptHeadroom());
pdiv._();
}
}
[31/50] [abbrv] hadoop git commit: YARN-3415. Non-AM containers can
be counted towards amResourceUsage of a fairscheduler queue (Zhihai Xu via
Sandy Ryza)
Posted by zj...@apache.org.
YARN-3415. Non-AM containers can be counted towards amResourceUsage of a fairscheduler queue (Zhihai Xu via Sandy Ryza)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e75307b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e75307b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e75307b2
Branch: refs/heads/YARN-2928
Commit: e75307b282014f73b435c0d4f6e5399af4ba128c
Parents: 97092e4
Author: Sandy Ryza <sa...@cloudera.com>
Authored: Thu Apr 2 13:56:08 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:14 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../scheduler/fair/FSAppAttempt.java | 31 +++++++-----
.../scheduler/fair/FSLeafQueue.java | 5 +-
.../scheduler/fair/FairScheduler.java | 6 ---
.../scheduler/fair/TestFairScheduler.java | 50 ++++++++++++++++----
5 files changed, 64 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bc8d06a..18a3c37 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -188,6 +188,9 @@ Release 2.8.0 - UNRELEASED
YARN-3425. NPE from RMNodeLabelsManager.serviceStop when
NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda)
+ YARN-3415. Non-AM containers can be counted towards amResourceUsage of a
+ Fair Scheduler queue (Zhihai Xu via Sandy Ryza)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 46617ff..f0d1ed1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -523,8 +523,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Inform the node
node.allocateContainer(allocatedContainer);
- // If this container is used to run AM, update the leaf queue's AM usage
- if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
+ // If not running unmanaged, the first container we allocate is always
+ // the AM. Set the amResource for this app and update the leaf queue's AM
+ // usage
+ if (!isAmRunning() && !getUnmanagedAM()) {
+ setAMResource(container.getResource());
getQueue().addAMResourceUsage(container.getResource());
setAmRunning(true);
}
@@ -551,6 +554,19 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved);
}
+ // Check the AM resource usage for the leaf queue
+ if (!isAmRunning() && !getUnmanagedAM()) {
+ List<ResourceRequest> ask = appSchedulingInfo.getAllResourceRequests();
+ if (ask.isEmpty() || !getQueue().canRunAppAM(
+ ask.get(0).getCapability())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping allocation because maxAMShare limit would " +
+ "be exceeded");
+ }
+ return Resources.none();
+ }
+ }
+
Collection<Priority> prioritiesToTry = (reserved) ?
Arrays.asList(node.getReservedContainer().getReservedPriority()) :
getPriorities();
@@ -567,17 +583,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
addSchedulingOpportunity(priority);
- // Check the AM resource usage for the leaf queue
- if (getLiveContainers().size() == 0 && !getUnmanagedAM()) {
- if (!getQueue().canRunAppAM(getAMResource())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping allocation because maxAMShare limit would " +
- "be exceeded");
- }
- return Resources.none();
- }
- }
-
ResourceRequest rackLocalRequest = getResourceRequest(priority,
node.getRackName());
ResourceRequest localRequest = getResourceRequest(priority,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index c49a323..04dbd2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -124,8 +124,9 @@ public class FSLeafQueue extends FSQueue {
writeLock.unlock();
}
- // Update AM resource usage if needed
- if (runnable && app.isAmRunning() && app.getAMResource() != null) {
+ // Update AM resource usage if needed. If isAMRunning is true, we're not
+ // running an unmanaged AM.
+ if (runnable && app.isAmRunning()) {
Resources.subtractFrom(amResourceUsage, app.getAMResource());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 04c7f70..a6c5416 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -901,12 +901,6 @@ public class FairScheduler extends
// Record container allocation start time
application.recordContainerRequestTime(getClock().getTime());
- // Set amResource for this app
- if (!application.getUnmanagedAM() && ask.size() == 1
- && application.getLiveContainers().isEmpty()) {
- application.setAMResource(ask.get(0).getCapability());
- }
-
// Release containers
releaseContainers(release, application);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e75307b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ff215cd..b5bfb8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtil
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -3548,8 +3549,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app3 = scheduler.getSchedulerApp(attId3);
scheduler.update();
scheduler.handle(updateEvent);
- assertEquals("Application3's AM requests 1024 MB memory",
- 1024, app3.getAMResource().getMemory());
+ assertEquals("Application3's AM resource shouldn't be updated",
+ 0, app3.getAMResource().getMemory());
assertEquals("Application3's AM should not be running",
0, app3.getLiveContainers().size());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
@@ -3574,6 +3575,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
0, app1.getLiveContainers().size());
assertEquals("Application3's AM should be running",
1, app3.getLiveContainers().size());
+ assertEquals("Application3's AM requests 1024 MB memory",
+ 1024, app3.getAMResource().getMemory());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemory());
@@ -3584,8 +3587,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app4 = scheduler.getSchedulerApp(attId4);
scheduler.update();
scheduler.handle(updateEvent);
- assertEquals("Application4's AM requests 2048 MB memory",
- 2048, app4.getAMResource().getMemory());
+ assertEquals("Application4's AM resource shouldn't be updated",
+ 0, app4.getAMResource().getMemory());
assertEquals("Application4's AM should not be running",
0, app4.getLiveContainers().size());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
@@ -3598,8 +3601,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app5 = scheduler.getSchedulerApp(attId5);
scheduler.update();
scheduler.handle(updateEvent);
- assertEquals("Application5's AM requests 2048 MB memory",
- 2048, app5.getAMResource().getMemory());
+ assertEquals("Application5's AM resource shouldn't be updated",
+ 0, app5.getAMResource().getMemory());
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
@@ -3631,6 +3634,33 @@ public class TestFairScheduler extends FairSchedulerTestBase {
0, app3.getLiveContainers().size());
assertEquals("Application5's AM should be running",
1, app5.getLiveContainers().size());
+ assertEquals("Application5's AM requests 2048 MB memory",
+ 2048, app5.getAMResource().getMemory());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // request non-AM container for app5
+ createSchedulingRequestExistingApplication(1024, 1, attId5);
+ assertEquals("Application5's AM should have 1 container",
+ 1, app5.getLiveContainers().size());
+ // complete AM container before non-AM container is allocated.
+ // spark application hit this situation.
+ RMContainer amContainer5 = (RMContainer)app5.getLiveContainers().toArray()[0];
+ ContainerExpiredSchedulerEvent containerExpired =
+ new ContainerExpiredSchedulerEvent(amContainer5.getContainerId());
+ scheduler.handle(containerExpired);
+ assertEquals("Application5's AM should have 0 container",
+ 0, app5.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ // non-AM container should be allocated
+ // check non-AM container allocation is not rejected
+ // due to queue MaxAMShare limitation.
+ assertEquals("Application5 should have 1 container",
+ 1, app5.getLiveContainers().size());
+ // check non-AM container allocation won't affect queue AmResourceUsage
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemory());
@@ -3643,8 +3673,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
assertEquals("Application6's AM should not be running",
0, app6.getLiveContainers().size());
- assertEquals("Application6's AM requests 2048 MB memory",
- 2048, app6.getAMResource().getMemory());
+ assertEquals("Application6's AM resource shouldn't be updated",
+ 0, app6.getAMResource().getMemory());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemory());
@@ -3748,8 +3778,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app2 = scheduler.getSchedulerApp(attId2);
scheduler.update();
scheduler.handle(updateEvent);
- assertEquals("Application2's AM requests 1024 MB memory",
- 1024, app2.getAMResource().getMemory());
+ assertEquals("Application2's AM resource shouldn't be updated",
+ 0, app2.getAMResource().getMemory());
assertEquals("Application2's AM should not be running",
0, app2.getLiveContainers().size());
assertEquals("Queue2's AM resource usage should be 0 MB memory",
[49/50] [abbrv] hadoop git commit: YARN-3435. AM container to be
allocated Appattempt AM container shown as null. Contributed by Bibin A
Chundatt
Posted by zj...@apache.org.
YARN-3435. AM container to be allocated Appattempt AM container shown as
null. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b09b406
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b09b406
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b09b406
Branch: refs/heads/YARN-2928
Commit: 0b09b406c6d09e031357ad84a810c9ddfe4a922d
Parents: 1069dce
Author: Xuan <xg...@apache.org>
Authored: Sun Apr 5 00:40:57 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:17 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b09b406/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 004044a..0c5aac8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -198,6 +198,9 @@ Release 2.8.0 - UNRELEASED
YARN-3415. Non-AM containers can be counted towards amResourceUsage of a
Fair Scheduler queue (Zhihai Xu via Sandy Ryza)
+ YARN-3435. AM container to be allocated Appattempt AM container shown as null.
+ (Bibin A Chundatt via xgong)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b09b406/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 1831920..506e31f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -209,7 +209,7 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
"AM Container:",
appAttempt.getAmContainerId() == null || containers == null
|| !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
- ? null : root_url("container", appAttempt.getAmContainerId()),
+ ? "N/A" : root_url("container", appAttempt.getAmContainerId()),
String.valueOf(appAttempt.getAmContainerId()))
._("Node:", node)
._(
[08/50] [abbrv] hadoop git commit: YARN-3248. Correct fix version
from branch-2.7 to branch-2.8 in the change log.
Posted by zj...@apache.org.
YARN-3248. Correct fix version from branch-2.7 to branch-2.8 in the change log.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f95a2de4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f95a2de4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f95a2de4
Branch: refs/heads/YARN-2928
Commit: f95a2de4a3a1b3846a82e7cd34b1140b7154c0f0
Parents: 40f362b
Author: Xuan <xg...@apache.org>
Authored: Wed Apr 1 04:32:11 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:10 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f95a2de4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 461f07d..c48de2c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED
YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring
from info to debug. (Anubhav Dhoot via ozawa)
+ YARN-3248. Display count of nodes blacklisted by apps in the web UI.
+ (Varun Vasudev via xgong)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
@@ -920,9 +923,6 @@ Release 2.7.0 - UNRELEASED
removing inconsistencies in the default values. (Junping Du and Karthik
Kambatla via vinodkv)
- YARN-3248. Display count of nodes blacklisted by apps in the web UI.
- (Varun Vasudev via xgong)
-
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
[33/50] [abbrv] hadoop git commit: YARN-2901. Add errors and warning
metrics page to RM, NM web UI. (Varun Vasudev via wangda)
Posted by zj...@apache.org.
YARN-2901. Add errors and warning metrics page to RM, NM web UI. (Varun Vasudev via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5d8402d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5d8402d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5d8402d
Branch: refs/heads/YARN-2928
Commit: d5d8402d943a141ab1497d2d0cb8371cd6d3dd78
Parents: 4f66d40
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Apr 2 17:23:20 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:15 2015 -0700
----------------------------------------------------------------------
.../src/main/conf/log4j.properties | 9 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../util/Log4jWarningErrorMetricsAppender.java | 447 +++++++++++++++++++
.../TestLog4jWarningErrorMetricsAppender.java | 260 +++++++++++
.../server/webapp/ErrorsAndWarningsBlock.java | 233 ++++++++++
.../server/nodemanager/webapp/NMController.java | 4 +
.../webapp/NMErrorsAndWarningsPage.java | 55 +++
.../server/nodemanager/webapp/NavBlock.java | 24 +-
.../server/nodemanager/webapp/WebServer.java | 1 +
.../server/resourcemanager/webapp/NavBlock.java | 27 +-
.../webapp/RMErrorsAndWarningsPage.java | 54 +++
.../server/resourcemanager/webapp/RMWebApp.java | 1 +
.../resourcemanager/webapp/RmController.java | 4 +
13 files changed, 1114 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 316c48e..3a0a3ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -266,3 +266,12 @@ log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log
#log4j.appender.nodemanagerrequestlog.RetainDays=3
+
+# Appender for viewing information for errors and warnings
+yarn.ewma.cleanupInterval=300
+yarn.ewma.messageAgeLimitSeconds=86400
+yarn.ewma.maxUniqueMessages=250
+log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender
+log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}
+log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}
+log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bcd2286..004044a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -157,6 +157,9 @@ Release 2.8.0 - UNRELEASED
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
(Varun Vasudev via xgong)
+ YARN-2901. Add errors and warning metrics page to RM, NM web UI.
+ (Varun Vasudev via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java
new file mode 100644
index 0000000..0366ae0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jWarningErrorMetricsAppender.java
@@ -0,0 +1,447 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+
+import java.util.*;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Log4jWarningErrorMetricsAppender extends AppenderSkeleton {
+
+ public static final String LOG_METRICS_APPENDER = "RM_LOG_METRICS_APPENDER";
+ static final int MAX_MESSAGE_SIZE = 2048;
+
+ static public class Element {
+ public Long count;
+ public Long timestampSeconds;
+
+ Element(Long count, Long timestampSeconds) {
+ this.count = count;
+ this.timestampSeconds = timestampSeconds;
+ }
+ }
+
+ static class PurgeElement implements Comparable<PurgeElement> {
+ String message;
+ Long timestamp;
+
+ PurgeElement(String message, Long timestamp) {
+ this.message = message;
+ this.timestamp = timestamp;
+ }
+
+ public int compareTo(PurgeElement e) {
+ if (e == null) {
+ throw new NullPointerException("Null element passed to compareTo");
+ }
+ int ret = this.timestamp.compareTo(e.timestamp);
+ if (ret != 0) {
+ return ret;
+ }
+ return this.message.compareTo(e.message);
+ }
+
+ @Override
+ public boolean equals(Object e) {
+ if (e == null || !(e instanceof PurgeElement)) {
+ return false;
+ }
+ if (e == this) {
+ return true;
+ }
+ PurgeElement el = (PurgeElement) e;
+ return (this.message.equals(el.message))
+ && (this.timestamp.equals(el.timestamp));
+ }
+
+ @Override
+ public int hashCode() {
+ return this.timestamp.hashCode();
+ }
+ }
+
+ Map<String, SortedMap<Long, Integer>> errors;
+ Map<String, SortedMap<Long, Integer>> warnings;
+ SortedMap<Long, Integer> errorsTimestampCount;
+ SortedMap<Long, Integer> warningsTimestampCount;
+ SortedSet<PurgeElement> errorsPurgeInformation;
+ SortedSet<PurgeElement> warningsPurgeInformation;
+
+ Timer cleanupTimer;
+ long cleanupInterval;
+ long messageAgeLimitSeconds;
+ int maxUniqueMessages;
+
+ final Object lock = new Object();
+
+ /**
+ * Create an appender to keep track of the errors and warnings logged by the
+ * system. It will keep purge messages older than 2 days. It will store upto
+ * the last 500 unique errors and the last 500 unique warnings. The thread to
+ * purge message will run every 5 minutes, unless the 500 message limit is hit
+ * earlier.
+ */
+ public Log4jWarningErrorMetricsAppender() {
+ this(5 * 60, 24 * 60 * 60, 250);
+ }
+
+ /**
+ * Create an appender to keep track of the errors and warnings logged by the
+ * system.
+ *
+ * @param cleanupIntervalSeconds
+ * the interval at which old messages are purged to prevent the
+ * message stores from growing unbounded
+ * @param messageAgeLimitSeconds
+ * the maximum age of a message in seconds before it is purged from
+ * the store
+ * @param maxUniqueMessages
+ * the maximum number of unique messages of each type we keep before
+ * we start purging
+ */
+ public Log4jWarningErrorMetricsAppender(int cleanupIntervalSeconds,
+ long messageAgeLimitSeconds, int maxUniqueMessages) {
+ super();
+ errors = new HashMap<>();
+ warnings = new HashMap<>();
+ errorsTimestampCount = new TreeMap<>();
+ warningsTimestampCount = new TreeMap<>();
+ errorsPurgeInformation = new TreeSet<>();
+ warningsPurgeInformation = new TreeSet<>();
+
+ cleanupTimer = new Timer();
+ cleanupInterval = cleanupIntervalSeconds * 1000;
+ cleanupTimer.schedule(new ErrorAndWarningsCleanup(), cleanupInterval);
+ this.messageAgeLimitSeconds = messageAgeLimitSeconds;
+ this.maxUniqueMessages = maxUniqueMessages;
+ this.setName(LOG_METRICS_APPENDER);
+ this.setThreshold(Level.WARN);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void append(LoggingEvent event) {
+ String message = event.getRenderedMessage();
+ String[] throwableStr = event.getThrowableStrRep();
+ if (throwableStr != null) {
+ message = message + "\n" + StringUtils.join("\n", throwableStr);
+ message =
+ org.apache.commons.lang.StringUtils.left(message, MAX_MESSAGE_SIZE);
+ }
+ int level = event.getLevel().toInt();
+
+ if (level == Level.WARN_INT || level == Level.ERROR_INT) {
+ // store second level information
+ Long eventTimeSeconds = event.getTimeStamp() / 1000;
+ Map<String, SortedMap<Long, Integer>> map;
+ SortedMap<Long, Integer> timestampsCount;
+ SortedSet<PurgeElement> purgeInformation;
+ if (level == Level.WARN_INT) {
+ map = warnings;
+ timestampsCount = warningsTimestampCount;
+ purgeInformation = warningsPurgeInformation;
+ } else {
+ map = errors;
+ timestampsCount = errorsTimestampCount;
+ purgeInformation = errorsPurgeInformation;
+ }
+ updateMessageDetails(message, eventTimeSeconds, map, timestampsCount,
+ purgeInformation);
+ }
+ }
+
+ private void updateMessageDetails(String message, Long eventTimeSeconds,
+ Map<String, SortedMap<Long, Integer>> map,
+ SortedMap<Long, Integer> timestampsCount,
+ SortedSet<PurgeElement> purgeInformation) {
+ synchronized (lock) {
+ if (map.containsKey(message)) {
+ SortedMap<Long, Integer> tmp = map.get(message);
+ Long lastMessageTime = tmp.lastKey();
+ int value = 1;
+ if (tmp.containsKey(eventTimeSeconds)) {
+ value = tmp.get(eventTimeSeconds) + 1;
+ }
+ tmp.put(eventTimeSeconds, value);
+ purgeInformation.remove(new PurgeElement(message, lastMessageTime));
+ } else {
+ SortedMap<Long, Integer> value = new TreeMap<>();
+ value.put(eventTimeSeconds, 1);
+ map.put(message, value);
+ if (map.size() > maxUniqueMessages * 2) {
+ cleanupTimer.cancel();
+ cleanupTimer = new Timer();
+ cleanupTimer.schedule(new ErrorAndWarningsCleanup(), 0);
+ }
+ }
+ purgeInformation.add(new PurgeElement(message, eventTimeSeconds));
+ int newValue = 1;
+ if (timestampsCount.containsKey(eventTimeSeconds)) {
+ newValue = timestampsCount.get(eventTimeSeconds) + 1;
+ }
+ timestampsCount.put(eventTimeSeconds, newValue);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void close() {
+ cleanupTimer.cancel();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ /**
+ * Get the counts of errors in the time periods provided. Note that the counts
+ * provided by this function may differ from the ones provided by
+ * getErrorMessagesAndCounts since the message store is purged at regular
+ * intervals to prevent it from growing without bounds, while the store for
+ * the counts is purged less frequently.
+ *
+ * @param cutoffs
+ * list of timestamp cutoffs(in seconds) for which the counts are
+ * desired
+ * @return list of error counts in the time periods corresponding to cutoffs
+ */
+ public List<Integer> getErrorCounts(List<Long> cutoffs) {
+ return this.getCounts(errorsTimestampCount, cutoffs);
+ }
+
+ /**
+ * Get the counts of warnings in the time periods provided. Note that the
+ * counts provided by this function may differ from the ones provided by
+ * getWarningMessagesAndCounts since the message store is purged at regular
+ * intervals to prevent it from growing without bounds, while the store for
+ * the counts is purged less frequently.
+ *
+ * @param cutoffs
+ * list of timestamp cutoffs(in seconds) for which the counts are
+ * desired
+ * @return list of warning counts in the time periods corresponding to cutoffs
+ */
+ public List<Integer> getWarningCounts(List<Long> cutoffs) {
+ return this.getCounts(warningsTimestampCount, cutoffs);
+ }
+
+ private List<Integer> getCounts(SortedMap<Long, Integer> map,
+ List<Long> cutoffs) {
+ List<Integer> ret = new ArrayList<>();
+ Long largestCutoff = Collections.min(cutoffs);
+ for (int i = 0; i < cutoffs.size(); ++i) {
+ ret.add(0);
+ }
+ synchronized (lock) {
+ Map<Long, Integer> submap = map.tailMap(largestCutoff);
+ for (Map.Entry<Long, Integer> entry : submap.entrySet()) {
+ for (int i = 0; i < cutoffs.size(); ++i) {
+ if (entry.getKey() >= cutoffs.get(i)) {
+ int tmp = ret.get(i);
+ ret.set(i, tmp + entry.getValue());
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Get the errors and the number of occurrences for each of the errors for the
+ * time cutoffs provided. Note that the counts provided by this function may
+ * differ from the ones provided by getErrorCounts since the message store is
+ * purged at regular intervals to prevent it from growing without bounds,
+ * while the store for the counts is purged less frequently.
+ *
+ * @param cutoffs
+ * list of timestamp cutoffs(in seconds) for which the counts are
+ * desired
+ * @return list of maps corresponding for each cutoff provided; each map
+ * contains the error and the number of times the error occurred in
+ * the time period
+ */
+ public List<Map<String, Element>>
+ getErrorMessagesAndCounts(List<Long> cutoffs) {
+ return this.getElementsAndCounts(errors, cutoffs, errorsPurgeInformation);
+ }
+
+ /**
+ * Get the warning and the number of occurrences for each of the warnings for
+ * the time cutoffs provided. Note that the counts provided by this function
+ * may differ from the ones provided by getWarningCounts since the message
+ * store is purged at regular intervals to prevent it from growing without
+ * bounds, while the store for the counts is purged less frequently.
+ *
+ * @param cutoffs
+ * list of timestamp cutoffs(in seconds) for which the counts are
+ * desired
+ * @return list of maps corresponding for each cutoff provided; each map
+ * contains the warning and the number of times the error occurred in
+ * the time period
+ */
+ public List<Map<String, Element>> getWarningMessagesAndCounts(
+ List<Long> cutoffs) {
+ return this.getElementsAndCounts(warnings, cutoffs, warningsPurgeInformation);
+ }
+
+ private List<Map<String, Element>> getElementsAndCounts(
+ Map<String, SortedMap<Long, Integer>> map, List<Long> cutoffs,
+ SortedSet<PurgeElement> purgeInformation) {
+ if (purgeInformation.size() > maxUniqueMessages) {
+ ErrorAndWarningsCleanup cleanup = new ErrorAndWarningsCleanup();
+ long cutoff = Time.now() - (messageAgeLimitSeconds * 1000);
+ cutoff = (cutoff / 1000);
+ cleanup.cleanupMessages(map, purgeInformation, cutoff, maxUniqueMessages);
+ }
+ List<Map<String, Element>> ret = new ArrayList<>(cutoffs.size());
+ for (int i = 0; i < cutoffs.size(); ++i) {
+ ret.add(new HashMap<String, Element>());
+ }
+ synchronized (lock) {
+ for (Map.Entry<String, SortedMap<Long, Integer>> element : map.entrySet()) {
+ for (int i = 0; i < cutoffs.size(); ++i) {
+ Map<String, Element> retMap = ret.get(i);
+ SortedMap<Long, Integer> qualifyingTimes =
+ element.getValue().tailMap(cutoffs.get(i));
+ long count = 0;
+ for (Map.Entry<Long, Integer> entry : qualifyingTimes.entrySet()) {
+ count += entry.getValue();
+ }
+ if (!qualifyingTimes.isEmpty()) {
+ retMap.put(element.getKey(),
+ new Element(count, qualifyingTimes.lastKey()));
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
+ // getters and setters for log4j
+ public long getCleanupInterval() {
+ return cleanupInterval;
+ }
+
+ public void setCleanupInterval(long cleanupInterval) {
+ this.cleanupInterval = cleanupInterval;
+ }
+
+ public long getMessageAgeLimitSeconds() {
+ return messageAgeLimitSeconds;
+ }
+
+ public void setMessageAgeLimitSeconds(long messageAgeLimitSeconds) {
+ this.messageAgeLimitSeconds = messageAgeLimitSeconds;
+ }
+
+ public int getMaxUniqueMessages() {
+ return maxUniqueMessages;
+ }
+
+ public void setMaxUniqueMessages(int maxUniqueMessages) {
+ this.maxUniqueMessages = maxUniqueMessages;
+ }
+
+ class ErrorAndWarningsCleanup extends TimerTask {
+
+ @Override
+ public void run() {
+ long cutoff = Time.now() - (messageAgeLimitSeconds * 1000);
+ cutoff = (cutoff / 1000);
+ cleanupMessages(errors, errorsPurgeInformation, cutoff, maxUniqueMessages);
+ cleanupMessages(warnings, warningsPurgeInformation, cutoff,
+ maxUniqueMessages);
+ cleanupCounts(errorsTimestampCount, cutoff);
+ cleanupCounts(warningsTimestampCount, cutoff);
+ try {
+ cleanupTimer.schedule(new ErrorAndWarningsCleanup(), cleanupInterval);
+ } catch (IllegalStateException ie) {
+ // don't do anything since new timer is already scheduled
+ }
+ }
+
+ void cleanupMessages(Map<String, SortedMap<Long, Integer>> map,
+ SortedSet<PurgeElement> purgeInformation, long cutoff,
+ int mapTargetSize) {
+
+ PurgeElement el = new PurgeElement("", cutoff);
+ synchronized (lock) {
+ SortedSet<PurgeElement> removeSet = purgeInformation.headSet(el);
+ Iterator<PurgeElement> it = removeSet.iterator();
+ while (it.hasNext()) {
+ PurgeElement p = it.next();
+ map.remove(p.message);
+ it.remove();
+ }
+
+ // don't keep more mapTargetSize keys
+ if (purgeInformation.size() > mapTargetSize) {
+ Object[] array = purgeInformation.toArray();
+ int cutoffIndex = purgeInformation.size() - mapTargetSize;
+ for (int i = 0; i < cutoffIndex; ++i) {
+ PurgeElement p = (PurgeElement) array[i];
+ map.remove(p.message);
+ purgeInformation.remove(p);
+ }
+ }
+ }
+ }
+
+ void cleanupCounts(SortedMap<Long, Integer> map, long cutoff) {
+ synchronized (lock) {
+ Iterator<Map.Entry<Long, Integer>> it = map.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, Integer> element = it.next();
+ if (element.getKey() < cutoff) {
+ it.remove();
+ }
+ }
+ }
+ }
+ }
+
+ // helper function
+ public static Log4jWarningErrorMetricsAppender findAppender() {
+ Enumeration appenders = Logger.getRootLogger().getAllAppenders();
+ while(appenders.hasMoreElements()) {
+ Object obj = appenders.nextElement();
+ if(obj instanceof Log4jWarningErrorMetricsAppender) {
+ return (Log4jWarningErrorMetricsAppender) obj;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java
new file mode 100644
index 0000000..61d4c4c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Time;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class TestLog4jWarningErrorMetricsAppender {
+
+ Log4jWarningErrorMetricsAppender appender;
+ Log logger = LogFactory.getLog(TestLog4jWarningErrorMetricsAppender.class);
+ List<Long> cutoff = new ArrayList<>();
+
+ void setupAppender(int cleanupIntervalSeconds, long messageAgeLimitSeconds,
+ int maxUniqueMessages) {
+ removeAppender();
+ appender =
+ new Log4jWarningErrorMetricsAppender(cleanupIntervalSeconds,
+ messageAgeLimitSeconds, maxUniqueMessages);
+ Logger.getRootLogger().addAppender(appender);
+ }
+
+ void removeAppender() {
+ Logger.getRootLogger().removeAppender(appender);
+ }
+
+ void logMessages(Level level, String message, int count) {
+ for (int i = 0; i < count; ++i) {
+ switch (level.toInt()) {
+ case Level.FATAL_INT:
+ logger.fatal(message);
+ break;
+ case Level.ERROR_INT:
+ logger.error(message);
+ break;
+ case Level.WARN_INT:
+ logger.warn(message);
+ break;
+ case Level.INFO_INT:
+ logger.info(message);
+ break;
+ case Level.DEBUG_INT:
+ logger.debug(message);
+ break;
+ case Level.TRACE_INT:
+ logger.trace(message);
+ break;
+ }
+ }
+ }
+
+ @Test
+ public void testPurge() throws Exception {
+ setupAppender(2, 1, 1);
+ logMessages(Level.ERROR, "test message 1", 1);
+ cutoff.clear();
+ cutoff.add(0L);
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).get(0)
+ .size());
+ Thread.sleep(2000);
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(0, appender.getErrorMessagesAndCounts(cutoff).get(0)
+ .size());
+
+ setupAppender(2, 1000, 2);
+
+ logMessages(Level.ERROR, "test message 1", 3);
+ logMessages(Level.ERROR, "test message 2", 2);
+
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(5, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).get(0)
+ .size());
+ logMessages(Level.ERROR, "test message 3", 3);
+ Thread.sleep(2000);
+ Assert.assertEquals(8, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).get(0)
+ .size());
+ }
+
+ @Test
+ public void testErrorCounts() throws Exception {
+ cutoff.clear();
+ setupAppender(100, 100, 100);
+ cutoff.add(0L);
+ logMessages(Level.ERROR, "test message 1", 2);
+ logMessages(Level.ERROR, "test message 2", 3);
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getWarningCounts(cutoff).size());
+ Assert.assertEquals(5, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert
+ .assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
+ Thread.sleep(1000);
+ cutoff.add(Time.now() / 1000);
+ logMessages(Level.ERROR, "test message 3", 2);
+ Assert.assertEquals(2, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(2, appender.getWarningCounts(cutoff).size());
+ Assert.assertEquals(7, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(2, appender.getErrorCounts(cutoff).get(1).longValue());
+ Assert
+ .assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
+ Assert
+ .assertEquals(0, appender.getWarningCounts(cutoff).get(1).longValue());
+ }
+
+ @Test
+ public void testWarningCounts() throws Exception {
+ cutoff.clear();
+ setupAppender(100, 100, 100);
+ cutoff.add(0L);
+ logMessages(Level.WARN, "test message 1", 2);
+ logMessages(Level.WARN, "test message 2", 3);
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getWarningCounts(cutoff).size());
+ Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert
+ .assertEquals(5, appender.getWarningCounts(cutoff).get(0).longValue());
+ Thread.sleep(1000);
+ cutoff.add(Time.now() / 1000);
+ logMessages(Level.WARN, "test message 3", 2);
+ Assert.assertEquals(2, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(2, appender.getWarningCounts(cutoff).size());
+ Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(1).longValue());
+ Assert
+ .assertEquals(7, appender.getWarningCounts(cutoff).get(0).longValue());
+ Assert
+ .assertEquals(2, appender.getWarningCounts(cutoff).get(1).longValue());
+ }
+
+ @Test
+ public void testWarningMessages() throws Exception {
+ cutoff.clear();
+ setupAppender(100, 100, 100);
+ cutoff.add(0L);
+ logMessages(Level.WARN, "test message 1", 2);
+ logMessages(Level.WARN, "test message 2", 3);
+ Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
+ Map<String, Log4jWarningErrorMetricsAppender.Element> errorsMap =
+ appender.getErrorMessagesAndCounts(cutoff).get(0);
+ Map<String, Log4jWarningErrorMetricsAppender.Element> warningsMap =
+ appender.getWarningMessagesAndCounts(cutoff).get(0);
+ Assert.assertEquals(0, errorsMap.size());
+ Assert.assertEquals(2, warningsMap.size());
+ Assert.assertTrue(warningsMap.containsKey("test message 1"));
+ Assert.assertTrue(warningsMap.containsKey("test message 2"));
+ Log4jWarningErrorMetricsAppender.Element msg1Info = warningsMap.get("test message 1");
+ Log4jWarningErrorMetricsAppender.Element msg2Info = warningsMap.get("test message 2");
+ Assert.assertEquals(2, msg1Info.count.intValue());
+ Assert.assertEquals(3, msg2Info.count.intValue());
+ Thread.sleep(1000);
+ cutoff.add(Time.now() / 1000);
+ logMessages(Level.WARN, "test message 3", 2);
+ Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).size());
+ Assert.assertEquals(2, appender.getWarningMessagesAndCounts(cutoff).size());
+ errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(0);
+ warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(0);
+ Assert.assertEquals(0, errorsMap.size());
+ Assert.assertEquals(3, warningsMap.size());
+ Assert.assertTrue(warningsMap.containsKey("test message 3"));
+ errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(1);
+ warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(1);
+ Assert.assertEquals(0, errorsMap.size());
+ Assert.assertEquals(1, warningsMap.size());
+ Assert.assertTrue(warningsMap.containsKey("test message 3"));
+ Log4jWarningErrorMetricsAppender.Element msg3Info = warningsMap.get("test message 3");
+ Assert.assertEquals(2, msg3Info.count.intValue());
+ }
+
+ @Test
+ public void testErrorMessages() throws Exception {
+ cutoff.clear();
+ setupAppender(100, 100, 100);
+ cutoff.add(0L);
+ logMessages(Level.ERROR, "test message 1", 2);
+ logMessages(Level.ERROR, "test message 2", 3);
+ Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
+ Map<String, Log4jWarningErrorMetricsAppender.Element> errorsMap =
+ appender.getErrorMessagesAndCounts(cutoff).get(0);
+ Map<String, Log4jWarningErrorMetricsAppender.Element> warningsMap =
+ appender.getWarningMessagesAndCounts(cutoff).get(0);
+ Assert.assertEquals(2, errorsMap.size());
+ Assert.assertEquals(0, warningsMap.size());
+ Assert.assertTrue(errorsMap.containsKey("test message 1"));
+ Assert.assertTrue(errorsMap.containsKey("test message 2"));
+ Log4jWarningErrorMetricsAppender.Element msg1Info = errorsMap.get("test message 1");
+ Log4jWarningErrorMetricsAppender.Element msg2Info = errorsMap.get("test message 2");
+ Assert.assertEquals(2, msg1Info.count.intValue());
+ Assert.assertEquals(3, msg2Info.count.intValue());
+ Thread.sleep(1000);
+ cutoff.add(Time.now() / 1000);
+ logMessages(Level.ERROR, "test message 3", 2);
+ Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).size());
+ Assert.assertEquals(2, appender.getWarningMessagesAndCounts(cutoff).size());
+ errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(0);
+ warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(0);
+ Assert.assertEquals(3, errorsMap.size());
+ Assert.assertEquals(0, warningsMap.size());
+ Assert.assertTrue(errorsMap.containsKey("test message 3"));
+ errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(1);
+ warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(1);
+ Assert.assertEquals(1, errorsMap.size());
+ Assert.assertEquals(0, warningsMap.size());
+ Assert.assertTrue(errorsMap.containsKey("test message 3"));
+ Log4jWarningErrorMetricsAppender.Element msg3Info = errorsMap.get("test message 3");
+ Assert.assertEquals(2, msg3Info.count.intValue());
+ }
+
+ @Test
+ public void testInfoDebugTrace() {
+ cutoff.clear();
+ setupAppender(100, 100, 100);
+ cutoff.add(0L);
+ logMessages(Level.INFO, "test message 1", 2);
+ logMessages(Level.DEBUG, "test message 2", 2);
+ logMessages(Level.TRACE, "test message 3", 2);
+ Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
+ Assert.assertEquals(1, appender.getWarningCounts(cutoff).size());
+ Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+ Assert
+ .assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
+ Assert.assertEquals(0, appender.getErrorMessagesAndCounts(cutoff).get(0)
+ .size());
+ Assert.assertEquals(0, appender.getWarningMessagesAndCounts(cutoff).get(0)
+ .size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
new file mode 100644
index 0000000..773a9e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import com.google.inject.Inject;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ErrorsAndWarningsBlock extends HtmlBlock {
+
+ long cutoffPeriodSeconds;
+
+ @Inject
+ ErrorsAndWarningsBlock(ViewContext ctx) {
+ super(ctx);
+ // default is to show all errors and warnings
+ cutoffPeriodSeconds = Time.now() / 1000;
+ String value = ctx.requestContext().get("cutoff", "");
+ try {
+ cutoffPeriodSeconds = Integer.parseInt(value);
+ if (cutoffPeriodSeconds <= 0) {
+ cutoffPeriodSeconds = Time.now() / 1000;
+ }
+ } catch (NumberFormatException ne) {
+ cutoffPeriodSeconds = Time.now() / 1000;
+ }
+ }
+
+ @Override
+ protected void render(Block html) {
+ Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
+
+ if (log instanceof Log4JLogger) {
+ html._(ErrorMetrics.class);
+ html._(WarningMetrics.class);
+ html.div().button().$onclick("reloadPage()").b("View data for the last ")
+ ._().select().$id("cutoff").option().$value("60")._("1 min")._()
+ .option().$value("300")._("5 min")._().option().$value("900")
+ ._("15 min")._().option().$value("3600")._("1 hour")._().option()
+ .$value("21600")._("6 hours")._().option().$value("43200")
+ ._("12 hours")._().option().$value("86400")._("24 hours")._()._()._();
+
+ String script = "function reloadPage() {"
+ + " var timePeriod = $(\"#cutoff\").val();"
+ + " document.location.href = '/cluster/errors-and-warnings?cutoff=' + timePeriod"
+ + "}";
+ script = script
+ + "; function toggleContent(element) {"
+ + " $(element).parent().siblings('.toggle-content').fadeToggle();"
+ + "}";
+
+ html.script().$type("text/javascript")._(script)._();
+
+ html.style(".toggle-content { display: none; }");
+
+ Log4jWarningErrorMetricsAppender appender =
+ Log4jWarningErrorMetricsAppender.findAppender();
+ if (appender == null) {
+ return;
+ }
+ List<Long> cutoff = new ArrayList<>();
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet>> errorsTable =
+ html.table("#messages").thead().tr().th(".message", "Message")
+ .th(".type", "Type").th(".count", "Count")
+ .th(".lasttime", "Latest Message Time")._()._().tbody();
+
+ // cutoff has to be in seconds
+ cutoff.add((Time.now() - cutoffPeriodSeconds * 1000) / 1000);
+ List<Map<String, Log4jWarningErrorMetricsAppender.Element>> errorsData =
+ appender.getErrorMessagesAndCounts(cutoff);
+ List<Map<String, Log4jWarningErrorMetricsAppender.Element>> warningsData =
+ appender.getWarningMessagesAndCounts(cutoff);
+ Map<String, List<Map<String, Log4jWarningErrorMetricsAppender.Element>>> sources =
+ new HashMap<>();
+ sources.put("Error", errorsData);
+ sources.put("Warning", warningsData);
+
+ int maxDisplayLength = 80;
+ for (Map.Entry<String, List<Map<String, Log4jWarningErrorMetricsAppender.Element>>> source : sources
+ .entrySet()) {
+ String type = source.getKey();
+ List<Map<String, Log4jWarningErrorMetricsAppender.Element>> data =
+ source.getValue();
+ if (data.size() > 0) {
+ Map<String, Log4jWarningErrorMetricsAppender.Element> map = data.get(0);
+ for (Map.Entry<String, Log4jWarningErrorMetricsAppender.Element> entry : map
+ .entrySet()) {
+ String message = entry.getKey();
+ Hamlet.TR<Hamlet.TBODY<Hamlet.TABLE<Hamlet>>> row =
+ errorsTable.tr();
+ Hamlet.TD<Hamlet.TR<Hamlet.TBODY<Hamlet.TABLE<Hamlet>>>> cell =
+ row.td();
+ if (message.length() > maxDisplayLength || message.contains("\n")) {
+ String displayMessage = entry.getKey().split("\n")[0];
+ if (displayMessage.length() > maxDisplayLength) {
+ displayMessage = displayMessage.substring(0, maxDisplayLength);
+ }
+
+ cell.pre().a().$href("#").$onclick("toggleContent(this);")
+ .$style("white-space: pre")._(displayMessage)._()._().div()
+ .$class("toggle-content").pre()._(message)._()._()._();
+ } else {
+ cell.pre()._(message)._()._();
+ }
+ Log4jWarningErrorMetricsAppender.Element ele = entry.getValue();
+ row.td(type).td(String.valueOf(ele.count))
+ .td(Times.format(ele.timestampSeconds * 1000))._();
+ }
+ }
+ }
+ errorsTable._()._();
+ }
+ }
+
+ public static class MetricsBase extends HtmlBlock {
+ List<Long> cutoffs;
+ List<Integer> values;
+ String tableHeading;
+ Log4jWarningErrorMetricsAppender appender;
+
+ MetricsBase(ViewContext ctx) {
+ super(ctx);
+ cutoffs = new ArrayList<>();
+
+ // cutoff has to be in seconds
+ long now = Time.now();
+ cutoffs.add((now - 60 * 1000) / 1000);
+ cutoffs.add((now - 300 * 1000) / 1000);
+ cutoffs.add((now - 900 * 1000) / 1000);
+ cutoffs.add((now - 3600 * 1000) / 1000);
+ cutoffs.add((now - 21600 * 1000) / 1000);
+ cutoffs.add((now - 43200 * 1000) / 1000);
+ cutoffs.add((now - 84600 * 1000) / 1000);
+
+ Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
+ if (log instanceof Log4JLogger) {
+ appender =
+ Log4jWarningErrorMetricsAppender.findAppender();
+ }
+ }
+
+ List<Long> getCutoffs() {
+ return this.cutoffs;
+ }
+
+ @Override
+ protected void render(Block html) {
+ Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
+ if (log instanceof Log4JLogger) {
+ Hamlet.DIV<Hamlet> div =
+ html.div().$class("metrics").$style("padding-bottom: 20px");
+ div.h3(tableHeading).table("#metricsoverview").thead()
+ .$class("ui-widget-header").tr().th().$class("ui-state-default")
+ ._("Last 1 minute")._().th().$class("ui-state-default")
+ ._("Last 5 minutes")._().th().$class("ui-state-default")
+ ._("Last 15 minutes")._().th().$class("ui-state-default")
+ ._("Last 1 hour")._().th().$class("ui-state-default")
+ ._("Last 6 hours")._().th().$class("ui-state-default")
+ ._("Last 12 hours")._().th().$class("ui-state-default")
+ ._("Last 24 hours")._()._()._().tbody().$class("ui-widget-content")
+ .tr().td(String.valueOf(values.get(0)))
+ .td(String.valueOf(values.get(1))).td(String.valueOf(values.get(2)))
+ .td(String.valueOf(values.get(3))).td(String.valueOf(values.get(4)))
+ .td(String.valueOf(values.get(5))).td(String.valueOf(values.get(6)))
+ ._()._()._();
+ div._();
+ }
+ }
+ }
+
+ public static class ErrorMetrics extends MetricsBase {
+
+ @Inject
+ ErrorMetrics(ViewContext ctx) {
+ super(ctx);
+ tableHeading = "Error Metrics";
+ }
+
+ @Override
+ protected void render(Block html) {
+ if (appender == null) {
+ return;
+ }
+ values = appender.getErrorCounts(getCutoffs());
+ super.render(html);
+ }
+ }
+
+ public static class WarningMetrics extends MetricsBase {
+
+ @Inject
+ WarningMetrics(ViewContext ctx) {
+ super(ctx);
+ tableHeading = "Warning Metrics";
+ }
+
+ @Override
+ protected void render(Block html) {
+ if (appender == null) {
+ return;
+ }
+ values = appender.getWarningCounts(getCutoffs());
+ super.render(html);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
index 86e2505..097532f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
@@ -75,6 +75,10 @@ public class NMController extends Controller implements YarnWebParams {
render(ContainerPage.class);
}
+ public void errorsAndWarnings() {
+ render(NMErrorsAndWarningsPage.class);
+ }
+
public void logs() {
String containerIdStr = $(CONTAINER_ID);
ContainerId containerId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMErrorsAndWarningsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMErrorsAndWarningsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMErrorsAndWarningsPage.java
new file mode 100644
index 0000000..7475c4d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMErrorsAndWarningsPage.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.server.webapp.ErrorsAndWarningsBlock;
+import org.apache.hadoop.yarn.webapp.view.HtmlPage;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+
+public class NMErrorsAndWarningsPage extends NMView {
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return ErrorsAndWarningsBlock.class;
+ }
+
+ @Override
+ protected void preHead(HtmlPage.Page.HTML<HtmlPage._> html) {
+ commonPreHead(html);
+ String title = "Errors and Warnings in the NodeManager";
+ setTitle(title);
+ String tableId = "messages";
+ set(DATATABLES_ID, tableId);
+ set(initID(DATATABLES, tableId), tablesInit());
+ setTableStyles(html, tableId, ".message {width:50em}",
+ ".count {width:8em}", ".lasttime {width:16em}");
+ }
+
+ private String tablesInit() {
+ StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ b.append("{'sType': 'string', 'aTargets': [ 0 ]}");
+ b.append(", {'sType': 'string', 'bSearchable': true, 'aTargets': [ 1 ]}");
+ b.append(", {'sType': 'numeric', 'bSearchable': false, 'aTargets': [ 2 ]}");
+ b.append(", {'sType': 'date', 'aTargets': [ 3 ] }]");
+ b.append(", aaSorting: [[3, 'desc']]}");
+ return b.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
index b8a10f1..857a4f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
@@ -18,8 +18,13 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -36,10 +41,19 @@ public class NavBlock extends HtmlBlock implements YarnWebParams {
@Override
protected void render(Block html) {
+
+ boolean addErrorsAndWarningsLink = false;
+ Log log = LogFactory.getLog(NMErrorsAndWarningsPage.class);
+ if (log instanceof Log4JLogger) {
+ Log4jWarningErrorMetricsAppender appender = Log4jWarningErrorMetricsAppender.findAppender();
+ if (appender != null) {
+ addErrorsAndWarningsLink = true;
+ }
+ }
String RMWebAppURL =
- WebAppUtils.getResolvedRemoteRMWebAppURLWithScheme(this.conf);
- html
+ WebAppUtils.getResolvedRMWebAppURLWithScheme(this.conf);
+ Hamlet.UL<Hamlet.DIV<Hamlet>> ul = html
.div("#nav")
.h3()._("ResourceManager")._()
.ul()
@@ -59,7 +73,11 @@ public class NavBlock extends HtmlBlock implements YarnWebParams {
.li().a("/conf", "Configuration")._()
.li().a("/logs", "Local logs")._()
.li().a("/stacks", "Server stacks")._()
- .li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
+ .li().a("/jmx?qry=Hadoop:*", "Server metrics")._();
+ if (addErrorsAndWarningsLink) {
+ ul.li().a(url("errors-and-warnings"), "Errors/Warnings")._();
+ }
+ ul._()._();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
index fdff480..2b6b549 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
@@ -125,6 +125,7 @@ public class WebServer extends AbstractService {
route(
pajoin("/containerlogs", CONTAINER_ID, APP_OWNER, CONTAINER_LOG_TYPE),
NMController.class, "logs");
+ route("/errors-and-warnings", NMController.class, "errorsAndWarnings");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
index 48df391..7a8ab8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
@@ -18,7 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI;
@@ -28,6 +32,15 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
public class NavBlock extends HtmlBlock {
@Override public void render(Block html) {
+ boolean addErrorsAndWarningsLink = false;
+ Log log = LogFactory.getLog(RMErrorsAndWarningsPage.class);
+ if (log instanceof Log4JLogger) {
+ Log4jWarningErrorMetricsAppender appender =
+ Log4jWarningErrorMetricsAppender.findAppender();
+ if (appender != null) {
+ addErrorsAndWarningsLink = true;
+ }
+ }
UL<DIV<Hamlet>> mainList = html.
div("#nav").
h3("Cluster").
@@ -44,13 +57,17 @@ public class NavBlock extends HtmlBlock {
li().a(url("apps", state.toString()), state.toString())._();
}
subAppsList._()._();
- mainList.
+ UL<DIV<Hamlet>> tools = mainList.
li().a(url("scheduler"), "Scheduler")._()._().
- h3("Tools").
- ul().
- li().a("/conf", "Configuration")._().
+ h3("Tools").ul();
+ tools.li().a("/conf", "Configuration")._().
li().a("/logs", "Local logs")._().
li().a("/stacks", "Server stacks")._().
- li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
+ li().a("/jmx?qry=Hadoop:*", "Server metrics")._();
+
+ if (addErrorsAndWarningsLink) {
+ tools.li().a(url("errors-and-warnings"), "Errors/Warnings")._();
+ }
+ tools._()._();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMErrorsAndWarningsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMErrorsAndWarningsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMErrorsAndWarningsPage.java
new file mode 100644
index 0000000..216deeb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMErrorsAndWarningsPage.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.server.webapp.ErrorsAndWarningsBlock;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+
+public class RMErrorsAndWarningsPage extends RmView {
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return ErrorsAndWarningsBlock.class;
+ }
+
+ @Override
+ protected void preHead(Page.HTML<_> html) {
+ commonPreHead(html);
+ String title = "Errors and Warnings in the ResourceManager";
+ setTitle(title);
+ String tableId = "messages";
+ set(DATATABLES_ID, tableId);
+ set(initID(DATATABLES, tableId), tablesInit());
+ setTableStyles(html, tableId, ".message {width:50em}",
+ ".count {width:8em}", ".lasttime {width:16em}");
+ }
+
+ private String tablesInit() {
+ StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ b.append("{'sType': 'string', 'aTargets': [ 0 ]}");
+ b.append(", {'sType': 'string', 'bSearchable': true, 'aTargets': [ 1 ]}");
+ b.append(", {'sType': 'numeric', 'bSearchable': false, 'aTargets': [ 2 ]}");
+ b.append(", {'sType': 'date', 'aTargets': [ 3 ] }]");
+ b.append(", aaSorting: [[3, 'desc']]}");
+ return b.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 4189053..86300ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -67,6 +67,7 @@ public class RMWebApp extends WebApp implements YarnWebParams {
route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), RmController.class,
"appattempt");
route(pajoin("/container", CONTAINER_ID), RmController.class, "container");
+ route("/errors-and-warnings", RmController.class, "errorsAndWarnings");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d8402d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
index cc5232e..c8e3c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
@@ -105,4 +105,8 @@ public class RmController extends Controller {
setTitle("Node Labels");
render(NodeLabelsPage.class);
}
+
+ public void errorsAndWarnings() {
+ render(RMErrorsAndWarningsPage.class);
+ }
}
[14/50] [abbrv] hadoop git commit: HADOOP-11787.
OpensslSecureRandom.c pthread_threadid_np usage signature is wrong on 32-bit
Mac. Contributed by Kiran Kumar M R.
Posted by zj...@apache.org.
HADOOP-11787. OpensslSecureRandom.c pthread_threadid_np usage signature is wrong on 32-bit Mac. Contributed by Kiran Kumar M R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/188b7d0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/188b7d0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/188b7d0a
Branch: refs/heads/YARN-2928
Commit: 188b7d0a3cadf3d776f65481c226095a7f4ca611
Parents: 777f01b
Author: cnauroth <cn...@apache.org>
Authored: Wed Apr 1 11:40:09 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:11 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188b7d0a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 13d74fd..111fb5e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1184,6 +1184,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11754. RM fails to start in non-secure mode due to authentication
filter failure. (wheat9)
+ HADOOP-11787. OpensslSecureRandom.c pthread_threadid_np usage signature is
+ wrong on 32-bit Mac. (Kiran Kumar M R via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188b7d0a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
index 8f0c06d..26e1fa6 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
@@ -286,7 +286,9 @@ static unsigned long pthreads_thread_id(void)
#elif defined(__sun)
thread_id = (unsigned long)pthread_self();
#elif defined(__APPLE__)
- (void)pthread_threadid_np(pthread_self(), &thread_id);
+ __uint64_t tmp_thread_id;
+ (void)pthread_threadid_np(pthread_self(), &tmp_thread_id);
+ thread_id = (unsigned long)tmp_thread_id;
#else
#error "Platform not supported"
#endif
[07/50] [abbrv] hadoop git commit: YARN-3425. NPE from
RMNodeLabelsManager.serviceStop when NodeLabelsManager.serviceInit failed.
(Bibin A Chundatt via wangda)
Posted by zj...@apache.org.
YARN-3425. NPE from RMNodeLabelsManager.serviceStop when NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a920ab07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a920ab07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a920ab07
Branch: refs/heads/YARN-2928
Commit: a920ab0779e9de764ba80d4d2c53b7cb6f6d6d36
Parents: f95a2de
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Apr 1 10:14:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:10 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a920ab07/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c48de2c..6fbc26e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -185,6 +185,9 @@ Release 2.8.0 - UNRELEASED
YARN-3412. RM tests should use MockRM where possible. (kasha)
+ YARN-3425. NPE from RMNodeLabelsManager.serviceStop when
+ NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a920ab07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index a5e2756..fe38164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -258,7 +258,9 @@ public class CommonNodeLabelsManager extends AbstractService {
// for UT purpose
protected void stopDispatcher() {
AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
- asyncDispatcher.stop();
+ if (null != asyncDispatcher) {
+ asyncDispatcher.stop();
+ }
}
@Override
[46/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken
JDiff support in Hadoop 2. Contributed by Li Lu.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml
new file mode 100644
index 0000000..b105253
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.6.0.xml
@@ -0,0 +1,45596 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Mar 30 15:27:52 PDT 2015 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-core 2.6.0"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/classes:/Users/llu/hadoop-common/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home/lib/tools.jar:/Users/llu/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/llu/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/llu/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/llu/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/llu/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/llu/.m2/repository/common
s-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/llu/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/llu/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/llu/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/Users/llu/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/llu/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/llu/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/llu/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/llu/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/llu/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/
Users/llu/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/llu/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/llu/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/llu/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/llu/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/llu/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/llu/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/llu/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/llu/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/llu/.m2/repository/commons-conf
iguration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/llu/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/llu/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/llu/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/llu/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/llu/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/llu/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/llu/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/llu/.m2/repository/org/apache/ant/ant/1.8.1/ant-1.8.1.jar:/Users/llu/.m2/repository/org/apache/ant/ant-launcher/1.8.1/
ant-launcher-1.8.1.jar:/Users/llu/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/llu/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/llu/hadoop-common/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/llu/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/llu/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/llu/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/llu/.m2/repository/org/apache/curator
/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/llu/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/llu/.m2/repository/org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar:/Users/llu/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/llu/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/llu/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/llu/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar -sourcepath /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/src/main/java -apidir /Users/llu/hadoop-common/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname hadoop-core 2.6.0 -->
+<package name="org.apache.hadoop">
+ <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+ <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Constructs exception with the specified detail message.
+ @param message detailed message.]]>
+ </doc>
+ </constructor>
+ <doc>
+ <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+ <!-- start interface org.apache.hadoop.conf.Configurable -->
+ <interface name="Configurable" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="setConf"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Set the configuration to be used by this object.]]>
+ </doc>
+ </method>
+ <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return the configuration used by this object.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.conf.Configurable -->
+ <!-- start class org.apache.hadoop.conf.Configuration -->
+ <class name="Configuration" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Iterable"/>
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <constructor name="Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration.]]>
+ </doc>
+ </constructor>
+ <constructor name="Configuration" type="boolean"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration where the behavior of reading from the default
+ resources can be turned off.
+
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files.
+ @param loadDefaults specifies whether to load from the default files]]>
+ </doc>
+ </constructor>
+ <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration with the same settings cloned from another.
+
+ @param other the configuration from which to clone settings.]]>
+ </doc>
+ </constructor>
+ <method name="addDeprecations"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+ <doc>
+ <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless. It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context. If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas The deprecations to add.]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKeys" type="java.lang.String[]"/>
+ <param name="customMessage" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If a key is deprecated in favor of multiple keys, they are all treated as
+ aliases of each other, and setting any one of them resets all the others
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKeys
+ @param customMessage
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKey" type="java.lang.String"/>
+ <param name="customMessage" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKey
+ @param customMessage]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKeys" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If a key is deprecated in favor of multiple keys, they are all treated as
+ aliases of each other, and setting any one of them resets all the others
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKey" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+ </doc>
+ </method>
+ <method name="isDeprecated" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <doc>
+ <![CDATA[checks whether the given <code>key</code> is deprecated.
+
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and
+ <code>false</code> otherwise.]]>
+ </doc>
+ </method>
+ <method name="setDeprecatedProperties"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+ </doc>
+ </method>
+ <method name="addDefaultResource"
+ abstract="false" native="false" synchronized="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a default resource. Resources are loaded in the order of the resources
+ added.
+ @param name file name. File should be present in the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param name resource to be added, the classpath is examined for a file
+ with that name.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="url" type="java.net.URL"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param url url of the resource to be added, the local filesystem is
+ examined directly to find the resource, without referring to
+ the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param file file-path of resource to be added, the local filesystem is
+ examined directly to find the resource, without referring to
+ the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.InputStream"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ WARNING: The contents of the InputStream will be cached, by this method.
+ So use this sparingly because it does increase the memory consumption.
+
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next. After it is read the stream will be
+ closed.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.InputStream"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+ </doc>
+ </method>
+ <method name="reloadConfiguration"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added
+ resources, and final parameters. This will make the resources to
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+ </doc>
+ </method>
+ <method name="get" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+
+ Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ before being returned.
+
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property,
+ or null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="getTrimmed" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ <code>null</code> if no such property exists.
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+
+ Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ before being returned.
+
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property,
+ or null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="getTrimmed" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ <code>defaultValue</code> if no such property exists.
+ See @{Configuration#getTrimmed} for more details.
+
+ @param name the property name.
+ @param defaultValue the property default value.
+ @return the value of the <code>name</code> or defaultValue
+ if it is not set.]]>
+ </doc>
+ </method>
+ <method name="getRaw" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is
+ deprecated, it returns the value of the first key which replaces
+ the deprecated key and is not null.
+
+ @param name the property name.
+ @return the value of the <code>name</code> property or
+ its replacing property and null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the <code>value</code> of the <code>name</code> property. If
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <param name="source" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the <code>value</code> of the <code>name</code> property. If
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+ </doc>
+ </method>
+ <method name="unset"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Unset a previously set property.]]>
+ </doc>
+ </method>
+ <method name="setIfUnset"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+ </doc>
+ </method>
+ <method name="get" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property
+ doesn't exist.]]>
+ </doc>
+ </method>
+ <method name="getInt" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="int"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getInts" return="int[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+
+ If no such property exists, an empty array is returned.
+
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+ <code>int</code> values]]>
+ </doc>
+ </method>
+ <method name="setInt"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="int"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="getLong" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getLongBytes" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setLong"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="getFloat" return="float"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="float"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setFloat"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="float"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="getDouble" return="double"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="double"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setDouble"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="double"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="getBoolean" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="boolean"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setBoolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="boolean"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="setBooleanIfUnset"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="boolean"/>
+ <doc>
+ <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+ </doc>
+ </method>
+ <method name="setEnum"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="T"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(<name>, value.toString())</code>.
+ @param name property name
+ @param value new value]]>
+ </doc>
+ </method>
+ <method name="getEnum" return="T"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="T"/>
+ <doc>
+ <![CDATA[Return value matching this enumerated type.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided]]>
+ </doc>
+ </method>
+ <method name="setTimeDuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ <param name="unit" type="java.util.concurrent.TimeUnit"/>
+ <doc>
+ <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(<name>, value + <time suffix>)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+ </doc>
+ </method>
+ <method name="getTimeDuration" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <param name="unit" type="java.util.concurrent.TimeUnit"/>
+ <doc>
+ <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+ a number]]>
+ </doc>
+ </method>
+ <method name="getPattern" return="java.util.regex.Pattern"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.util.regex.Pattern"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+ </doc>
+ </method>
+ <method name="setPattern"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="pattern" type="java.util.regex.Pattern"/>
+ <doc>
+ <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+ </doc>
+ </method>
+ <method name="getPropertySources" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Gets information about why a property was set. Typically this is the
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise,
+ returns a list of the sources of the resource. The older sources are
+ the first ones in the list. So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+ </doc>
+ </method>
+ <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+ </doc>
+ </method>
+ <method name="getStringCollection" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ a collection of <code>String</code>s.
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+ </doc>
+ </method>
+ <method name="getStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s.
+ If no such property is specified then <code>null</code> is returned.
+
+ @param name property name.
+ @return property value as an array of <code>String</code>s,
+ or <code>null</code>.]]>
+ </doc>
+ </method>
+ <method name="getStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s,
+ or default value.]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStringCollection" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s,
+ or empty array.]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s,
+ or default value.]]>
+ </doc>
+ </method>
+ <method name="setStrings"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="values" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Set the array of string values for the <code>name</code> property as
+ as comma delimited values.
+
+ @param name property name.
+ @param values The values]]>
+ </doc>
+ </method>
+ <method name="getPassword" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password]]>
+ </doc>
+ </method>
+ <method name="getPasswordFromCredentialProviders" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getPasswordFromConfig" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+ </doc>
+ </method>
+ <method name="getSocketAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="hostProperty" type="java.lang.String"/>
+ <param name="addressProperty" type="java.lang.String"/>
+ <param name="defaultAddressValue" type="java.lang.String"/>
+ <param name="defaultPort" type="int"/>
+ <doc>
+ <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ </doc>
+ </method>
+ <method name="getSocketAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultAddress" type="java.lang.String"/>
+ <param name="defaultPort" type="int"/>
+ <doc>
+ <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ </doc>
+ </method>
+ <method name="setSocketAddr"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+ </doc>
+ </method>
+ <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="hostProperty" type="java.lang.String"/>
+ <param name="addressProperty" type="java.lang.String"/>
+ <param name="defaultAddressValue" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>. The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address. This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+ </doc>
+ </method>
+ <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>. The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+ </doc>
+ </method>
+ <method name="getClassByName" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+ <doc>
+ <![CDATA[Load a class by name.
+
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+ </doc>
+ </method>
+ <method name="getClassByNameOrNull" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+ </doc>
+ </method>
+ <method name="getClasses" return="java.lang.Class[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class[]"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ @param name the class name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ An exception is thrown if the returned class does not implement the named
+ interface.
+
+ @param name the class name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getInstances" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+ <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+ </doc>
+ </method>
+ <method name="setClass"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="theClass" type="java.lang.Class"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to the name of a
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+
+ An exception is thrown if <code>theClass</code> does not implement the
+ interface <code>xface</code>.
+
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+ </doc>
+ </method>
+ <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dirsProp" type="java.lang.String"/>
+ <param name="path" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code. If the selected
+ directory does not exist, an attempt is made to create it.
+
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+ </doc>
+ </method>
+ <method name="getFile" return="java.io.File"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dirsProp" type="java.lang.String"/>
+ <param name="path" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code. If the selected
+ directory does not exist, an attempt is made to create it.
+
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+ </doc>
+ </method>
+ <method name="getResource" return="java.net.URL"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the {@link URL} for the named resource.
+
+ @param name resource name.
+ @return the url for the named resource.]]>
+ </doc>
+ </method>
+ <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+ </doc>
+ </method>
+ <method name="getConfResourceAsReader" return="java.io.Reader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+ </doc>
+ </method>
+ <method name="getFinalParameters" return="java.util.Set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+ </doc>
+ </method>
+ <method name="getProps" return="java.util.Properties"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+ </doc>
+ </method>
+ <method name="clear"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Clears all keys from the configuration.]]>
+ </doc>
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code>
+ key-value pairs in the configuration.
+
+ @return an iterator over the entries.]]>
+ </doc>
+ </method>
+ <method name="writeXml"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.OutputStream"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+
+ @param out the output stream to write to.]]>
+ </doc>
+ </method>
+ <method name="writeXml"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.Writer"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link Writer}.
+
+ @param out the writer to write to.]]>
+ </doc>
+ </method>
+ <method name="dumpConfiguration"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="out" type="java.io.Writer"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Writes out all the parameters and their properties (final and resource) to
+ the given {@link Writer}
+ The format of the output would be
+ { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
+ key2.isFinal,key2.resource}... ] }
+ It does not output the parameters of the configuration object which is
+ loaded from an input stream.
+ @param out the Writer to write to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getClassLoader" return="java.lang.ClassLoader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+ </doc>
+ </method>
+ <method name="setClassLoader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="classLoader" type="java.lang.ClassLoader"/>
+ <doc>
+ <![CDATA[Set the class loader that will be used to load the various objects.
+
+ @param classLoader the new class loader.]]>
+ </doc>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setQuietMode"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="quietmode" type="boolean"/>
+ <doc>
+ <![CDATA[Set the quietness-mode.
+
+ In the quiet-mode, error and informational messages might not be logged.
+
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+ to turn it off.]]>
+ </doc>
+ </method>
+ <method name="main"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="args" type="java.lang.String[]"/>
+ <exception name="Exception" type="java.lang.Exception"/>
+ <doc>
+ <![CDATA[For debugging. List non-default properties to the terminal and exit.]]>
+ </doc>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getValByRegex" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="regex" type="java.lang.String"/>
+ <doc>
+ <![CDATA[get keys matching the the regex
+ @param regex
+ @return Map<String,String> with matching keys]]>
+ </doc>
+ </method>
+ <method name="dumpDeprecatedKeys"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="hasWarnedDeprecation" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Provides access to configuration parameters.
+
+ <h4 id="Resources">Resources</h4>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>,
+ then the classpath is examined for a file with that name. If named by a
+ <code>Path</code>, then the local filesystem is examined directly, without
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>.
+ Once a resource declares a value final, no subsequently-loaded
+ resource can alter that value.
+ For example, one might define a final parameter with:
+ <tt><pre>
+ <property>
+ <name>dfs.hosts.include</name>
+ <value>/etc/hadoop/conf/hosts.include</value>
+ <b><final>true</final></b>
+ </property></pre></tt>
+
+ Administrators typically define parameters as final in
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions:
+ <tt><pre>
+ <property>
+ <name>basedir</name>
+ <value>/user/${<i>user.name</i>}</value>
+ </property>
+
+ <property>
+ <name>tempdir</name>
+ <value>${<i>basedir</i>}/tmp</value>
+ </property></pre></tt>
+
+ When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ By default, warnings will be given to any deprecated configuration
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration -->
+ <!-- start class org.apache.hadoop.conf.Configuration.DeprecationDelta -->
+ <class name="Configuration.DeprecationDelta" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Configuration.DeprecationDelta" type="java.lang.String, java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="Configuration.DeprecationDelta" type="java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getKey" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getNewKeys" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getCustomMessage" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A pending addition to the global set of deprecated keys.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration.DeprecationDelta -->
+ <!-- start class org.apache.hadoop.conf.Configuration.IntegerRanges -->
+ <class name="Configuration.IntegerRanges" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Iterable"/>
+ <constructor name="Configuration.IntegerRanges"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="Configuration.IntegerRanges" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="isIncluded" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="int"/>
+ <doc>
+ <![CDATA[Is the given value in the set of ranges
+ @param value the value to check
+ @return is the value in the ranges?]]>
+ </doc>
+ </method>
+ <method name="isEmpty" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true if there are no values in this range, else false.]]>
+ </doc>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A class that represents a set of positive integer ranges. It parses
+ strings of the form: "2-3,5,7-" where ranges are separated by comma and
+ the lower/upper bounds are separated by dash. Either the lower or upper
+ bound may be omitted meaning all values up to or over. So the string
+ above means 2, 3, 5, and 7, 8, 9, ...]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration.IntegerRanges -->
+ <!-- start class org.apache.hadoop.conf.Configured -->
+ <class name="Configured" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Configurable"/>
+ <constructor name="Configured"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a Configured.]]>
+ </doc>
+ </constructor>
+ <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a Configured.]]>
+ </doc>
+ </constructor>
+ <method name="setConf"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configured -->
+ <!-- start class org.apache.hadoop.conf.ConfServlet.BadFormatException -->
+ <class name="ConfServlet.BadFormatException" extends="java.lang.Exception"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ConfServlet.BadFormatException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ConfServlet.BadFormatException -->
+ <!-- start interface org.apache.hadoop.conf.Reconfigurable -->
+ <interface name="Reconfigurable" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Configurable"/>
+ <method name="reconfigureProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[Change a configuration property on this object to the value specified.
+
+ Change a configuration property on this object to the value specified
+ and return the previous value that the configuration property was set to
+ (or null if it was not previously set). If newVal is null, set the property
+ to its default value;
+
+ If the property cannot be changed, throw a
+ {@link ReconfigurationException}.]]>
+ </doc>
+ </method>
+ <method name="isPropertyReconfigurable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Return whether a given property is changeable at run time.
+
+ If isPropertyReconfigurable returns true for a property,
+ then changeConf should not throw an exception when changing
+ this property.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurableProperties" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return all the properties that can be changed at run time.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Something whose {@link Configuration} can be changed at run time.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.conf.Reconfigurable -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurableBase -->
+ <class name="ReconfigurableBase" extends="org.apache.hadoop.conf.Configured"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Reconfigurable"/>
+ <constructor name="ReconfigurableBase"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a ReconfigurableBase.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurableBase" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a ReconfigurableBase with the {@link Configuration}
+ conf.]]>
+ </doc>
+ </constructor>
+ <method name="setReconfigurationUtil"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ru" type="org.apache.hadoop.conf.ReconfigurationUtil"/>
+ </method>
+ <method name="getChangedProperties" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newConf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="oldConf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="startReconfigurationTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Start a reconfiguration task to reload configuration in background.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurationTaskStatus" return="org.apache.hadoop.conf.ReconfigurationTaskStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="shutdownReconfigurationTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="reconfigureProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ This method makes the change to this objects {@link Configuration}
+ and calls reconfigurePropertyImpl to update internal data structures.
+ This method cannot be overridden, subclasses should instead override
+ reconfigureProperty.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurableProperties" return="java.util.Collection"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ Subclasses must override this.]]>
+ </doc>
+ </method>
+ <method name="isPropertyReconfigurable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ Subclasses may wish to override this with a more efficient implementation.]]>
+ </doc>
+ </method>
+ <method name="reconfigurePropertyImpl"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[Change a configuration property.
+
+ Subclasses must override this. This method applies the change to
+ all internal data structures derived from the configuration property
+ that is being changed. If this object owns other Reconfigurable objects
+ reconfigureProperty should be called recursively to make sure that
+ to make sure that the configuration of these objects is updated.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Utility base class for implementing the Reconfigurable interface.
+
+ Subclasses should override reconfigurePropertyImpl to change individual
+ properties and getReconfigurableProperties to get all properties that
+ can be changed at run time.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurableBase -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationException -->
+ <class name="ReconfigurationException" extends="java.lang.Exception"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurationException" type="java.lang.String, java.lang.String, java.lang.String, java.lang.Throwable"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurationException" type="java.lang.String, java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <method name="getProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get property that cannot be changed.]]>
+ </doc>
+ </method>
+ <method name="getNewValue" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get value to which property was supposed to be changed.]]>
+ </doc>
+ </method>
+ <method name="getOldValue" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get old value of property that cannot be changed.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Exception indicating that configuration property cannot be changed
+ at run time.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurationException -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationServlet -->
+ <class name="ReconfigurationServlet" extends="javax.servlet.http.HttpServlet"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationServlet"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="init"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ </method>
+ <method name="doGet"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="req" type="javax.servlet.http.HttpServletRequest"/>
+ <param name="resp" type="javax.servlet.http.HttpServletResponse"/>
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="doPost"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="req" type="javax.servlet.http.HttpServletRequest"/>
+ <param name="resp" type="javax.servlet.http.HttpServletResponse"/>
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="CONF_SERVLET_RECONFIGURABLE_PREFIX" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[A servlet for changing a node's configuration.
+
+ Reloads the configuration file, verifies whether changes are
+ possible and asks the admin to approve the change.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurationServlet -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationTaskStatus -->
+ <class name="ReconfigurationTaskStatus" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationTaskStatus" type="long, long, java.util.Map"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="hasTask" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return true if
+ - A reconfiguration task has finished or
+ - an active reconfiguration task is run
<TRUNCATED>
[23/50] [abbrv] hadoop git commit: HDFS-7916. 'reportBadBlocks' from
datanodes to standby Node BPServiceActor goes for infinite loop (Contributed
by Vinayakumar B)
Posted by zj...@apache.org.
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c3ce1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c3ce1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c3ce1a
Branch: refs/heads/YARN-2928
Commit: b4c3ce1ac62e734cc0f725ffda1e1319d2a05aca
Parents: 0e6b61e
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Apr 2 08:12:00 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:13 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/ReportBadBlockAction.java | 8 ++++++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c3ce1a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 518df9f..80d958d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -412,6 +412,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7922. ShortCircuitCache#close is not releasing
ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
+ HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
+ goes for infinite loop (vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c3ce1a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index fd01a01..991b56d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -58,8 +59,11 @@ public class ReportBadBlockAction implements BPServiceActorAction {
dnArr, uuids, types) };
try {
- bpNamenode.reportBadBlocks(locatedBlock);
- } catch (IOException e){
+ bpNamenode.reportBadBlocks(locatedBlock);
+ } catch (StandbyException e) {
+ DataNode.LOG.warn("Failed to report bad block " + block
+ + " to standby namenode");
+ } catch (IOException e) {
throw new BPServiceActorActionException("Failed to report bad block "
+ block + " to namenode: ");
}
[37/50] [abbrv] hadoop git commit: HDFS-8034. Fix
TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay
for Windows. Contributed by Xiaoyu Yao.
Posted by zj...@apache.org.
HDFS-8034. Fix TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay for Windows. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/310c997b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/310c997b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/310c997b
Branch: refs/heads/YARN-2928
Commit: 310c997b56c53e660be31809cb4da74114824666
Parents: ff35b52
Author: cnauroth <cn...@apache.org>
Authored: Fri Apr 3 10:10:11 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:15 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 +++
.../hadoop/hdfs/TestDFSClientRetries.java | 33 +++++++++++---------
2 files changed, 22 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310c997b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 04eac75..366d8fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1366,6 +1366,10 @@ Release 2.7.0 - UNRELEASED
HDFS-7954. TestBalancer#testBalancerWithPinnedBlocks should not be executed
on Windows. (Xiaoyu Yao via szetszwo)
+ HDFS-8034. Fix
+ TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay
+ for Windows. (Xiaoyu Yao via cnauroth)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/310c997b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 3912774..c4258eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -1136,20 +1136,23 @@ public class TestDFSClientRetries {
throws Exception {
// test if DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY
// is not configured, verify DFSClient uses the default value 400.
- Configuration dfsConf = new HdfsConfiguration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(dfsConf).build();
- cluster.waitActive();
- NamenodeProtocols nn = cluster.getNameNodeRpc();
- DFSClient client = new DFSClient(null, nn, dfsConf, null);
- assertEquals(client.getConf().
- getBlockWriteLocateFollowingInitialDelayMs(), 400);
-
- // change DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY,
- // verify DFSClient uses the configured value 1000.
- dfsConf.setInt(DFSConfigKeys.
- DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY, 1000);
- client = new DFSClient(null, nn, dfsConf, null);
- assertEquals(client.getConf().
- getBlockWriteLocateFollowingInitialDelayMs(), 1000);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ cluster.waitActive();
+ NamenodeProtocols nn = cluster.getNameNodeRpc();
+ DFSClient client = new DFSClient(null, nn, conf, null);
+ assertEquals(client.getConf().
+ getBlockWriteLocateFollowingInitialDelayMs(), 400);
+
+ // change DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY,
+ // verify DFSClient uses the configured value 1000.
+ conf.setInt(DFSConfigKeys.
+ DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_INITIAL_DELAY_KEY, 1000);
+ client = new DFSClient(null, nn, conf, null);
+ assertEquals(client.getConf().
+ getBlockWriteLocateFollowingInitialDelayMs(), 1000);
+ } finally {
+ cluster.shutdown();
+ }
}
}
[35/50] [abbrv] hadoop git commit: HDFS-7811. Avoid recursive call
getStoragePolicyID in INodeFile#computeQuotaUsage. Contributed by Xiaoyu Yao
and Jing Zhao.
Posted by zj...@apache.org.
HDFS-7811. Avoid recursive call getStoragePolicyID in INodeFile#computeQuotaUsage. Contributed by Xiaoyu Yao and Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aae48dea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aae48dea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aae48dea
Branch: refs/heads/YARN-2928
Commit: aae48deac10c1ef26504997ae57d298bd6d6fd8d
Parents: c134f99
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Apr 2 21:20:30 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:15 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/namenode/FSImage.java | 14 +++++---
.../hadoop/hdfs/server/namenode/INode.java | 29 +++++++++++++++--
.../hdfs/server/namenode/INodeDirectory.java | 34 +++++++++++++-------
.../hadoop/hdfs/server/namenode/INodeFile.java | 13 ++++----
.../hadoop/hdfs/server/namenode/INodeMap.java | 4 +--
.../hdfs/server/namenode/INodeReference.java | 15 +++++----
.../hdfs/server/namenode/INodeSymlink.java | 2 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 7 ++--
9 files changed, 84 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 85c013e..04eac75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -877,6 +877,9 @@ Release 2.7.0 - UNRELEASED
HDFS-8008. Support client-side back off when the datanodes are congested.
(wheat9)
+ HDFS-7811. Avoid recursive call getStoragePolicyID in
+ INodeFile#computeQuotaUsage. (Xiaoyu Yao and jing9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 7454850..d62b952 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -856,23 +856,27 @@ public class FSImage implements Closeable {
*/
static void updateCountForQuota(BlockStoragePolicySuite bsps,
INodeDirectory root) {
- updateCountForQuotaRecursively(bsps, root, new QuotaCounts.Builder().build());
+ updateCountForQuotaRecursively(bsps, root.getStoragePolicyID(), root,
+ new QuotaCounts.Builder().build());
}
private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
- INodeDirectory dir, QuotaCounts counts) {
+ byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) {
final long parentNamespace = counts.getNameSpace();
final long parentStoragespace = counts.getStorageSpace();
final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();
- dir.computeQuotaUsage4CurrentDirectory(bsps, counts);
+ dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts);
for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
+ final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
if (child.isDirectory()) {
- updateCountForQuotaRecursively(bsps, child.asDirectory(), counts);
+ updateCountForQuotaRecursively(bsps, childPolicyId,
+ child.asDirectory(), counts);
} else {
// file or symlink: count here to reduce recursive calls.
- child.computeQuotaUsage(bsps, counts, false);
+ child.computeQuotaUsage(bsps, childPolicyId, counts, false,
+ Snapshot.CURRENT_STATE_ID);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 586cce4..e629441 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -505,9 +505,14 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
/**
* Count subtree {@link Quota#NAMESPACE} and {@link Quota#STORAGESPACE} usages.
+ * Entry point for FSDirectory where blockStoragePolicyId is given its initial
+ * value.
*/
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps) {
- return computeQuotaUsage(bsps, new QuotaCounts.Builder().build(), true);
+ final byte storagePolicyId = isSymlink() ?
+ BlockStoragePolicySuite.ID_UNSPECIFIED : getStoragePolicyID();
+ return computeQuotaUsage(bsps, storagePolicyId,
+ new QuotaCounts.Builder().build(), true, Snapshot.CURRENT_STATE_ID);
}
/**
@@ -532,6 +537,7 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* <pre>
*
* @param bsps Block storage policy suite to calculate intended storage type usage
+ * @param blockStoragePolicyId block storage policy id of the current INode
* @param counts The subtree counts for returning.
* @param useCache Whether to use cached quota usage. Note that
* {@link WithName} node never uses cache for its subtree.
@@ -542,12 +548,15 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* @return The same objects as the counts parameter.
*/
public abstract QuotaCounts computeQuotaUsage(
- BlockStoragePolicySuite bsps,
+ BlockStoragePolicySuite bsps, byte blockStoragePolicyId,
QuotaCounts counts, boolean useCache, int lastSnapshotId);
public final QuotaCounts computeQuotaUsage(
BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache) {
- return computeQuotaUsage(bsps, counts, useCache, Snapshot.CURRENT_STATE_ID);
+ final byte storagePolicyId = isSymlink() ?
+ BlockStoragePolicySuite.ID_UNSPECIFIED : getStoragePolicyID();
+ return computeQuotaUsage(bsps, storagePolicyId, counts,
+ useCache, Snapshot.CURRENT_STATE_ID);
}
/**
@@ -708,6 +717,20 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
public abstract byte getLocalStoragePolicyID();
/**
+ * Get the storage policy ID while computing quota usage
+ * @param parentStoragePolicyId the storage policy ID of the parent directory
+ * @return the storage policy ID of this INode. Note that for an
+ * {@link INodeSymlink} we return {@link BlockStoragePolicySuite#ID_UNSPECIFIED}
+ * instead of throwing Exception
+ */
+ public byte getStoragePolicyIDForQuota(byte parentStoragePolicyId) {
+ byte localId = isSymlink() ?
+ BlockStoragePolicySuite.ID_UNSPECIFIED : getLocalStoragePolicyID();
+ return localId != BlockStoragePolicySuite.ID_UNSPECIFIED ?
+ localId : parentStoragePolicyId;
+ }
+
+ /**
* Breaks {@code path} into components.
* @return array of byte arrays each of which represents
* a single path component.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index dadb8c7..9f55fc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -45,6 +45,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import static org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite.ID_UNSPECIFIED;
+
/**
* Directory INode class.
*/
@@ -123,18 +125,18 @@ public class INodeDirectory extends INodeWithAdditionalFields
return (xattr.getValue())[0];
}
}
- return BlockStoragePolicySuite.ID_UNSPECIFIED;
+ return ID_UNSPECIFIED;
}
@Override
public byte getStoragePolicyID() {
byte id = getLocalStoragePolicyID();
- if (id != BlockStoragePolicySuite.ID_UNSPECIFIED) {
+ if (id != ID_UNSPECIFIED) {
return id;
}
// if it is unspecified, check its parent
return getParent() != null ? getParent().getStoragePolicyID() :
- BlockStoragePolicySuite.ID_UNSPECIFIED;
+ ID_UNSPECIFIED;
}
void setQuota(BlockStoragePolicySuite bsps, long nsQuota, long ssQuota, StorageType type) {
@@ -568,10 +570,11 @@ public class INodeDirectory extends INodeWithAdditionalFields
}
@Override
- public QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache,
+ public QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps,
+ byte blockStoragePolicyId, QuotaCounts counts, boolean useCache,
int lastSnapshotId) {
final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
-
+
// we are computing the quota usage for a specific snapshot here, i.e., the
// computation only includes files/directories that exist at the time of the
// given snapshot
@@ -579,7 +582,9 @@ public class INodeDirectory extends INodeWithAdditionalFields
&& !(useCache && isQuotaSet())) {
ReadOnlyList<INode> childrenList = getChildrenList(lastSnapshotId);
for (INode child : childrenList) {
- child.computeQuotaUsage(bsps, counts, useCache, lastSnapshotId);
+ final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
+ child.computeQuotaUsage(bsps, childPolicyId, counts, useCache,
+ lastSnapshotId);
}
counts.addNameSpace(1);
return counts;
@@ -591,28 +596,33 @@ public class INodeDirectory extends INodeWithAdditionalFields
return q.AddCurrentSpaceUsage(counts);
} else {
useCache = q != null && !q.isQuotaSet() ? false : useCache;
- return computeDirectoryQuotaUsage(bsps, counts, useCache, lastSnapshotId);
+ return computeDirectoryQuotaUsage(bsps, blockStoragePolicyId, counts,
+ useCache, lastSnapshotId);
}
}
private QuotaCounts computeDirectoryQuotaUsage(BlockStoragePolicySuite bsps,
- QuotaCounts counts, boolean useCache, int lastSnapshotId) {
+ byte blockStoragePolicyId, QuotaCounts counts, boolean useCache,
+ int lastSnapshotId) {
if (children != null) {
for (INode child : children) {
- child.computeQuotaUsage(bsps, counts, useCache, lastSnapshotId);
+ final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
+ child.computeQuotaUsage(bsps, childPolicyId, counts, useCache,
+ lastSnapshotId);
}
}
- return computeQuotaUsage4CurrentDirectory(bsps, counts);
+ return computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId,
+ counts);
}
/** Add quota usage for this inode excluding children. */
public QuotaCounts computeQuotaUsage4CurrentDirectory(
- BlockStoragePolicySuite bsps, QuotaCounts counts) {
+ BlockStoragePolicySuite bsps, byte storagePolicyId, QuotaCounts counts) {
counts.addNameSpace(1);
// include the diff list
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
if (sf != null) {
- sf.computeQuotaUsage4CurrentDirectory(bsps, counts);
+ sf.computeQuotaUsage4CurrentDirectory(bsps, storagePolicyId, counts);
}
return counts;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index a6f07f9..294dd14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite.ID_UNSPECIFIED;
import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.NO_SNAPSHOT_ID;
@@ -24,7 +25,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -393,7 +393,7 @@ public class INodeFile extends INodeWithAdditionalFields
@Override
public byte getStoragePolicyID() {
byte id = getLocalStoragePolicyID();
- if (id == BlockStoragePolicySuite.ID_UNSPECIFIED) {
+ if (id == ID_UNSPECIFIED) {
return this.getParent() != null ?
this.getParent().getStoragePolicyID() : id;
}
@@ -554,7 +554,8 @@ public class INodeFile extends INodeWithAdditionalFields
// derive the intended storage type usage for quota by storage type
@Override
public final QuotaCounts computeQuotaUsage(
- BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache,
+ BlockStoragePolicySuite bsps, byte blockStoragePolicyId,
+ QuotaCounts counts, boolean useCache,
int lastSnapshotId) {
long nsDelta = 1;
final long ssDeltaNoReplication;
@@ -583,8 +584,8 @@ public class INodeFile extends INodeWithAdditionalFields
counts.addNameSpace(nsDelta);
counts.addStorageSpace(ssDeltaNoReplication * replication);
- if (getStoragePolicyID() != BlockStoragePolicySuite.ID_UNSPECIFIED){
- BlockStoragePolicy bsp = bsps.getPolicy(getStoragePolicyID());
+ if (blockStoragePolicyId != ID_UNSPECIFIED){
+ BlockStoragePolicy bsp = bsps.getPolicy(blockStoragePolicyId);
List<StorageType> storageTypes = bsp.chooseStorageTypes(replication);
for (StorageType t : storageTypes) {
if (!t.supportTypeQuota()) {
@@ -618,7 +619,7 @@ public class INodeFile extends INodeWithAdditionalFields
counts.addContent(Content.LENGTH, fileLen);
counts.addContent(Content.DISKSPACE, storagespaceConsumed());
- if (getStoragePolicyID() != BlockStoragePolicySuite.ID_UNSPECIFIED){
+ if (getStoragePolicyID() != ID_UNSPECIFIED){
BlockStoragePolicy bsp = summary.getBlockStoragePolicySuite().
getPolicy(getStoragePolicyID());
List<StorageType> storageTypes = bsp.chooseStorageTypes(getFileReplication());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index cb270bf..fc1cbec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -103,8 +103,8 @@ public class INodeMap {
@Override
public QuotaCounts computeQuotaUsage(
- BlockStoragePolicySuite bsps, QuotaCounts counts,
- boolean useCache, int lastSnapshotId) {
+ BlockStoragePolicySuite bsps, byte blockStoragePolicyId,
+ QuotaCounts counts, boolean useCache, int lastSnapshotId) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
index eee50a5..b33a93c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
@@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -327,9 +326,10 @@ public abstract class INodeReference extends INode {
@Override
public QuotaCounts computeQuotaUsage(
- BlockStoragePolicySuite bsps,
+ BlockStoragePolicySuite bsps, byte blockStoragePolicyId,
QuotaCounts counts, boolean useCache, int lastSnapshotId) {
- return referred.computeQuotaUsage(bsps, counts, useCache, lastSnapshotId);
+ return referred.computeQuotaUsage(bsps, blockStoragePolicyId, counts,
+ useCache, lastSnapshotId);
}
@Override
@@ -512,7 +512,8 @@ public abstract class INodeReference extends INode {
ContentSummaryComputationContext summary) {
//only count storagespace for WithName
final QuotaCounts q = new QuotaCounts.Builder().build();
- computeQuotaUsage(summary.getBlockStoragePolicySuite(), q, false, lastSnapshotId);
+ computeQuotaUsage(summary.getBlockStoragePolicySuite(),
+ getStoragePolicyID(), q, false, lastSnapshotId);
summary.getCounts().addContent(Content.DISKSPACE, q.getStorageSpace());
summary.getCounts().addTypeSpaces(q.getTypeSpaces());
return summary;
@@ -520,7 +521,8 @@ public abstract class INodeReference extends INode {
@Override
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps,
- QuotaCounts counts, boolean useCache, int lastSnapshotId) {
+ byte blockStoragePolicyId, QuotaCounts counts, boolean useCache,
+ int lastSnapshotId) {
// if this.lastSnapshotId < lastSnapshotId, the rename of the referred
// node happened before the rename of its ancestor. This should be
// impossible since for WithName node we only count its children at the
@@ -535,7 +537,8 @@ public abstract class INodeReference extends INode {
// been updated by changes in the current tree.
int id = lastSnapshotId != Snapshot.CURRENT_STATE_ID ?
lastSnapshotId : this.lastSnapshotId;
- return referred.computeQuotaUsage(bsps, counts, false, id);
+ return referred.computeQuotaUsage(bsps, blockStoragePolicyId, counts,
+ false, id);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
index 120d0dc..21a9e4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
@@ -93,7 +93,7 @@ public class INodeSymlink extends INodeWithAdditionalFields {
@Override
public QuotaCounts computeQuotaUsage(
- BlockStoragePolicySuite bsps,
+ BlockStoragePolicySuite bsps, byte blockStoragePolicyId,
QuotaCounts counts, boolean useCache, int lastSnapshotId) {
counts.addNameSpace(1);
return counts;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae48dea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index d55332f..95f9d8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -641,10 +641,13 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
public QuotaCounts computeQuotaUsage4CurrentDirectory(
- BlockStoragePolicySuite bsps, QuotaCounts counts) {
+ BlockStoragePolicySuite bsps, byte storagePolicyId,
+ QuotaCounts counts) {
for(DirectoryDiff d : diffs) {
for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
- deleted.computeQuotaUsage(bsps, counts, false, Snapshot.CURRENT_STATE_ID);
+ final byte childPolicyId = deleted.getStoragePolicyIDForQuota(storagePolicyId);
+ deleted.computeQuotaUsage(bsps, childPolicyId, counts, false,
+ Snapshot.CURRENT_STATE_ID);
}
}
return counts;
[09/50] [abbrv] hadoop git commit: YARN-3424. Change logs for
ContainerMonitorImpl's resourse monitoring from info to debug. Contributed by
Anubhav Dhoot.
Posted by zj...@apache.org.
YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring from info to debug. Contributed by Anubhav Dhoot.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db2998ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db2998ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db2998ed
Branch: refs/heads/YARN-2928
Commit: db2998ed25c2331a8a42ef50c2ee4c8afdc2a65e
Parents: b62eb19
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Apr 1 17:44:25 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:10 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../monitor/ContainersMonitorImpl.java | 13 ++++++++-----
2 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2998ed/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index dab8116..f18cf30 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -147,6 +147,9 @@ Release 2.8.0 - UNRELEASED
YARN-3428. Debug log resources to be localized for a container. (kasha)
+ YARN-3424. Change logs for ContainerMonitorImpl's resourse monitoring
+ from info to debug. (Anubhav Dhoot via ozawa)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2998ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 6200efa..bfd6b48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -519,11 +519,14 @@ public class ContainersMonitorImpl extends AbstractService implements
long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
long vmemLimit = ptInfo.getVmemLimit();
long pmemLimit = ptInfo.getPmemLimit();
- LOG.info(String.format(
- "Memory usage of ProcessTree %s for container-id %s: ",
- pId, containerId.toString()) +
- formatUsageString(
- currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format(
+ "Memory usage of ProcessTree %s for container-id %s: ",
+ pId, containerId.toString()) +
+ formatUsageString(
+ currentVmemUsage, vmemLimit,
+ currentPmemUsage, pmemLimit));
+ }
// Add usage to container metrics
if (containerMetricsEnabled) {
[36/50] [abbrv] hadoop git commit: HADOOP-11800. Clean up some test
methods in TestCodec.java. Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
HADOOP-11800. Clean up some test methods in TestCodec.java. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff35b525
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff35b525
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff35b525
Branch: refs/heads/YARN-2928
Commit: ff35b525ac2a2c92d4d2e366deee80a9fb48f576
Parents: aae48de
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Apr 4 00:32:18 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:15 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/test/java/org/apache/hadoop/io/compress/TestCodec.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff35b525/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 495c42d..7dcf8c1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -494,6 +494,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11760. Fix typo of javadoc in DistCp. (Brahma Reddy Battula via
ozawa).
+ HADOOP-11800. Clean up some test methods in TestCodec.java.
+ (Brahma Reddy Battula via aajisaka)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff35b525/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
index 7246bf5..5d4af91 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
@@ -827,7 +827,7 @@ public class TestCodec {
br.close();
}
- public void testGzipCodecWrite(boolean useNative) throws IOException {
+ private void testGzipCodecWrite(boolean useNative) throws IOException {
// Create a gzipped file using a compressor from the CodecPool,
// and try to read it back via the regular GZIPInputStream.
@@ -894,7 +894,7 @@ public class TestCodec {
public void testGzipNativeCodecWrite() throws IOException {
testGzipCodecWrite(true);
}
-
+ @Test
public void testCodecPoolAndGzipDecompressor() {
// BuiltInZlibInflater should not be used as the GzipCodec decompressor.
// Assert that this is the case.
[05/50] [abbrv] hadoop git commit: HDFS-7671. hdfs user guide should
point to the common rack awareness doc. Contributed by Kai Sasaki.
Posted by zj...@apache.org.
HDFS-7671. hdfs user guide should point to the common rack awareness doc. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8af286f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8af286f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8af286f4
Branch: refs/heads/YARN-2928
Commit: 8af286f4262328228c801cf0b1e8e6e49e255d6d
Parents: 6916c77
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Apr 1 00:26:16 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:09 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/site/markdown/HdfsUserGuide.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af286f4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e8075e6..f3537b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -368,6 +368,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7944. Minor cleanup of BlockPoolManager#getAllNamenodeThreads.
(Arpit Agarwal)
+ HDFS-7671. hdfs user guide should point to the common rack awareness doc.
+ (Kai Sasaki via aajisaka)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8af286f4/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index 37fa4be..ffd8532 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -264,7 +264,7 @@ For command usage, see [balancer](./HDFSCommands.html#balancer).
Rack Awareness
--------------
-Typically large Hadoop clusters are arranged in racks and network traffic between different nodes with in the same rack is much more desirable than network traffic across the racks. In addition NameNode tries to place replicas of block on multiple racks for improved fault tolerance. Hadoop lets the cluster administrators decide which rack a node belongs to through configuration variable `net.topology.script.file.name`. When this script is configured, each node runs the script to determine its rack id. A default installation assumes all the nodes belong to the same rack. This feature and configuration is further described in PDF attached to [HADOOP-692](https://issues.apache.org/jira/browse/HADOOP-692).
+A HDFS cluster can recognize the topology of racks where each nodes are put. It is important to configure this topology in order to optimize the data capacity and usage. For more detail, please check the [rack awareness](../hadoop-common/RackAwareness.html) in common document.
Safemode
--------
[30/50] [abbrv] hadoop git commit: MAPREDUCE-6303. Read timeout when
retrying a fetch error can be fatal to a reducer. Contributed by Jason Lowe.
Posted by zj...@apache.org.
MAPREDUCE-6303. Read timeout when retrying a fetch error can be fatal to a reducer. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97092e40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97092e40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97092e40
Branch: refs/heads/YARN-2928
Commit: 97092e4059c73fc77335c262404a37abca6d9bb7
Parents: d92ba15
Author: Junping Du <ju...@apache.org>
Authored: Thu Apr 2 12:13:03 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:14 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../hadoop/mapreduce/task/reduce/Fetcher.java | 73 ++++++++++----------
.../mapreduce/task/reduce/TestFetcher.java | 33 +++++++++
3 files changed, 74 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97092e40/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index f5d2d1a..160c49c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -513,6 +513,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6285. ClientServiceDelegate should not retry upon
AuthenticationException. (Jonathan Eagles via ozawa)
+ MAPREDUCE-6303. Read timeout when retrying a fetch error can be fatal
+ to a reducer. (Jason Lowe via junping_du)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97092e40/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 3f40853..d867e4b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -258,6 +258,39 @@ class Fetcher<K,V> extends Thread {
closeConnection();
}
+ private DataInputStream openShuffleUrl(MapHost host,
+ Set<TaskAttemptID> remaining, URL url) {
+ DataInputStream input = null;
+
+ try {
+ setupConnectionsWithRetry(host, remaining, url);
+ if (stopped) {
+ abortConnect(host, remaining);
+ } else {
+ input = new DataInputStream(connection.getInputStream());
+ }
+ } catch (IOException ie) {
+ boolean connectExcpt = ie instanceof ConnectException;
+ ioErrs.increment(1);
+ LOG.warn("Failed to connect to " + host + " with " + remaining.size() +
+ " map outputs", ie);
+
+ // If connect did not succeed, just mark all the maps as failed,
+ // indirectly penalizing the host
+ scheduler.hostFailed(host.getHostName());
+ for(TaskAttemptID left: remaining) {
+ scheduler.copyFailed(left, host, false, connectExcpt);
+ }
+
+ // Add back all the remaining maps, WITHOUT marking them as failed
+ for(TaskAttemptID left: remaining) {
+ scheduler.putBackKnownMapOutput(host, left);
+ }
+ }
+
+ return input;
+ }
+
/**
* The crux of the matter...
*
@@ -286,38 +319,12 @@ class Fetcher<K,V> extends Thread {
Set<TaskAttemptID> remaining = new HashSet<TaskAttemptID>(maps);
// Construct the url and connect
- DataInputStream input = null;
URL url = getMapOutputURL(host, maps);
- try {
- setupConnectionsWithRetry(host, remaining, url);
-
- if (stopped) {
- abortConnect(host, remaining);
- return;
- }
- } catch (IOException ie) {
- boolean connectExcpt = ie instanceof ConnectException;
- ioErrs.increment(1);
- LOG.warn("Failed to connect to " + host + " with " + remaining.size() +
- " map outputs", ie);
-
- // If connect did not succeed, just mark all the maps as failed,
- // indirectly penalizing the host
- scheduler.hostFailed(host.getHostName());
- for(TaskAttemptID left: remaining) {
- scheduler.copyFailed(left, host, false, connectExcpt);
- }
-
- // Add back all the remaining maps, WITHOUT marking them as failed
- for(TaskAttemptID left: remaining) {
- scheduler.putBackKnownMapOutput(host, left);
- }
-
+ DataInputStream input = openShuffleUrl(host, remaining, url);
+ if (input == null) {
return;
}
- input = new DataInputStream(connection.getInputStream());
-
try {
// Loop through available map-outputs and fetch them
// On any error, faildTasks is not null and we exit
@@ -333,14 +340,10 @@ class Fetcher<K,V> extends Thread {
connection.disconnect();
// Get map output from remaining tasks only.
url = getMapOutputURL(host, remaining);
-
- // Connect with retry as expecting host's recovery take sometime.
- setupConnectionsWithRetry(host, remaining, url);
- if (stopped) {
- abortConnect(host, remaining);
+ input = openShuffleUrl(host, remaining, url);
+ if (input == null) {
return;
}
- input = new DataInputStream(connection.getInputStream());
}
}
@@ -591,7 +594,7 @@ class Fetcher<K,V> extends Thread {
// Retry is not timeout, let's do retry with throwing an exception.
if (currentTime - retryStartTime < this.fetchRetryTimeout) {
LOG.warn("Shuffle output from " + host.getHostName() +
- " failed, retry it.");
+ " failed, retry it.", ioe);
throw ioe;
} else {
// timeout, prepare to be failed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97092e40/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
index 929c0ae..723df17 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
@@ -388,6 +388,39 @@ public class TestFetcher {
anyBoolean(), anyBoolean());
}
+ @SuppressWarnings("unchecked")
+ @Test(timeout=10000)
+ public void testCopyFromHostWithRetryThenTimeout() throws Exception {
+ InMemoryMapOutput<Text, Text> immo = mock(InMemoryMapOutput.class);
+ Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(jobWithRetry,
+ id, ss, mm, r, metrics, except, key, connection);
+
+ String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key);
+
+ when(connection.getResponseCode()).thenReturn(200)
+ .thenThrow(new SocketTimeoutException("forced timeout"));
+ when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH))
+ .thenReturn(replyHash);
+ ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 10, 10, 1);
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ header.write(new DataOutputStream(bout));
+ ByteArrayInputStream in = new ByteArrayInputStream(bout.toByteArray());
+ when(connection.getInputStream()).thenReturn(in);
+ when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME))
+ .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+ when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION))
+ .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+ when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt()))
+ .thenReturn(immo);
+ doThrow(new IOException("forced error")).when(immo).shuffle(
+ any(MapHost.class), any(InputStream.class), anyLong(),
+ anyLong(), any(ShuffleClientMetrics.class), any(Reporter.class));
+
+ underTest.copyFromHost(host);
+ verify(allErrs).increment(1);
+ verify(ss).copyFailed(map1ID, host, false, false);
+ }
+
@Test
public void testCopyFromHostExtraBytes() throws Exception {
Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
[40/50] [abbrv] hadoop git commit: HDFS-8051. FsVolumeList#addVolume
should release volume reference if not put it into BlockScanner. (Lei (Eddy)
Xu via Colin P. McCabe)
Posted by zj...@apache.org.
HDFS-8051. FsVolumeList#addVolume should release volume reference if not put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e1ffb3ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e1ffb3ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e1ffb3ea
Branch: refs/heads/YARN-2928
Commit: e1ffb3ea615e830b4215fafd0b2c1d05a50d4b0c
Parents: 5864e88
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Apr 3 16:34:23 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:16 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../datanode/fsdataset/impl/FsVolumeList.java | 5 +++++
.../fsdataset/impl/TestFsVolumeList.java | 19 +++++++++++++++++++
3 files changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ffb3ea/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2d399a4..6fafec8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1376,6 +1376,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7996. After swapping a volume, BlockReceiver reports
ReplicaNotFoundException (Lei (Eddy) Xu via Colin P. McCabe)
+ HDFS-8051. FsVolumeList#addVolume should release volume reference if not
+ put it into BlockScanner. (Lei (Eddy) Xu via Colin P. McCabe)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ffb3ea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 4fddfb9..d87595c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.Time;
@@ -292,6 +293,10 @@ class FsVolumeList {
}
if (blockScanner != null) {
blockScanner.addVolumeScanner(ref);
+ } else {
+ // If the volume is not put into a volume scanner, it does not need to
+ // hold the reference.
+ IOUtils.cleanup(FsDatasetImpl.LOG, ref);
}
// If the volume is used to replace a failed volume, it needs to reset the
// volume failure info for this volume.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ffb3ea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 46189ba..eccff89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
public class TestFsVolumeList {
@@ -101,4 +102,22 @@ public class TestFsVolumeList {
// checkDirs() should ignore the 2nd volume since it is closed.
volumeList.checkDirs();
}
+
+ @Test
+ public void testReleaseVolumeRefIfNoBlockScanner() throws IOException {
+ FsVolumeList volumeList = new FsVolumeList(
+ Collections.<VolumeFailureInfo>emptyList(), null, blockChooser);
+ File volDir = new File(baseDir, "volume-0");
+ volDir.mkdirs();
+ FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir,
+ conf, StorageType.DEFAULT);
+ FsVolumeReference ref = volume.obtainReference();
+ volumeList.addVolume(ref);
+ try {
+ ref.close();
+ fail("Should throw exception because the reference is closed in "
+ + "VolumeList#addVolume().");
+ } catch (IllegalStateException e) {
+ }
+ }
}
[13/50] [abbrv] hadoop git commit: HDFS-7978. Add
LOG.isDebugEnabled() guard for some LOG.debug(..). Contributed by Walter Su.
Posted by zj...@apache.org.
HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..). Contributed by Walter Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9fbc73e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9fbc73e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9fbc73e
Branch: refs/heads/YARN-2928
Commit: b9fbc73e57f2d64cbf7109f232d9c670ca6d7df6
Parents: 188b7d0
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Apr 1 12:53:25 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:11 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/BlockReaderFactory.java | 24 +++++++++-----------
.../java/org/apache/hadoop/hdfs/HAUtil.java | 12 ++++++----
.../hdfs/server/datanode/BPServiceActor.java | 8 ++++---
.../datanode/fsdataset/impl/FsDatasetCache.java | 8 ++++---
.../server/namenode/FileJournalManager.java | 22 +++++++++++-------
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +++-
.../hdfs/shortcircuit/ShortCircuitCache.java | 4 +++-
.../tools/offlineImageViewer/FSImageLoader.java | 6 +++--
.../hadoop/hdfs/util/LightWeightHashSet.java | 6 +++--
.../org/apache/hadoop/hdfs/web/TokenAspect.java | 8 +++++--
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +++++++++----
12 files changed, 78 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cba53a3..435fdd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -373,6 +373,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8009. Signal congestion on the DataNode. (wheat9)
+ HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..).
+ (Walter Su via wang)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
index 1e915b2..8f33899 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
@@ -369,9 +369,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
return null;
}
if (clientContext.getDisableLegacyBlockReaderLocal()) {
- PerformanceAdvisory.LOG.debug(this + ": can't construct " +
- "BlockReaderLocalLegacy because " +
- "disableLegacyBlockReaderLocal is set.");
+ PerformanceAdvisory.LOG.debug("{}: can't construct " +
+ "BlockReaderLocalLegacy because " +
+ "disableLegacyBlockReaderLocal is set.", this);
return null;
}
IOException ioe = null;
@@ -410,8 +410,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
getPathInfo(inetSocketAddress, conf);
}
if (!pathInfo.getPathState().getUsableForShortCircuit()) {
- PerformanceAdvisory.LOG.debug(this + ": " + pathInfo + " is not " +
- "usable for short circuit; giving up on BlockReaderLocal.");
+ PerformanceAdvisory.LOG.debug("{}: {} is not usable for short circuit; " +
+ "giving up on BlockReaderLocal.", this, pathInfo);
return null;
}
ShortCircuitCache cache = clientContext.getShortCircuitCache();
@@ -426,11 +426,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
throw exc;
}
if (info.getReplica() == null) {
- if (LOG.isTraceEnabled()) {
- PerformanceAdvisory.LOG.debug(this + ": failed to get " +
- "ShortCircuitReplica. Cannot construct " +
- "BlockReaderLocal via " + pathInfo.getPath());
- }
+ PerformanceAdvisory.LOG.debug("{}: failed to get " +
+ "ShortCircuitReplica. Cannot construct " +
+ "BlockReaderLocal via {}", this, pathInfo.getPath());
return null;
}
return new BlockReaderLocal.Builder(conf).
@@ -610,9 +608,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
getPathInfo(inetSocketAddress, conf);
}
if (!pathInfo.getPathState().getUsableForDataTransfer()) {
- PerformanceAdvisory.LOG.debug(this + ": not trying to create a " +
- "remote block reader because the UNIX domain socket at " +
- pathInfo + " is not usable.");
+ PerformanceAdvisory.LOG.debug("{}: not trying to create a " +
+ "remote block reader because the UNIX domain socket at {}" +
+ " is not usable.", this, pathInfo);
return null;
}
if (LOG.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
index f91f709..0ee57c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
@@ -319,12 +319,16 @@ public class HAUtil {
buildTokenServicePrefixForLogicalUri(HdfsConstants.HDFS_URI_SCHEME)
+ "//" + specificToken.getService());
ugi.addToken(alias, specificToken);
- LOG.debug("Mapped HA service delegation token for logical URI " +
- haUri + " to namenode " + singleNNAddr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Mapped HA service delegation token for logical URI " +
+ haUri + " to namenode " + singleNNAddr);
+ }
}
} else {
- LOG.debug("No HA service delegation token found for logical URI " +
- haUri);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No HA service delegation token found for logical URI " +
+ haUri);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index df582f1..dd6f9ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -580,9 +580,11 @@ class BPServiceActor implements Runnable {
long createCost = createTime - startTime;
long sendCost = sendTime - createTime;
dn.getMetrics().addCacheReport(sendCost);
- LOG.debug("CacheReport of " + blockIds.size()
- + " block(s) took " + createCost + " msec to generate and "
- + sendCost + " msecs for RPC and NN processing");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CacheReport of " + blockIds.size()
+ + " block(s) took " + createCost + " msec to generate and "
+ + sendCost + " msecs for RPC and NN processing");
+ }
}
return cmd;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
index c6408e6..e0df0f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
@@ -319,9 +319,11 @@ public class FsDatasetCache {
mappableBlockMap.put(key,
new Value(prevValue.mappableBlock, State.UNCACHING));
if (deferred) {
- LOG.debug("{} is anchored, and can't be uncached now. Scheduling it " +
- "for uncaching in {} ",
- key, DurationFormatUtils.formatDurationHMS(revocationPollingMs));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} is anchored, and can't be uncached now. Scheduling it " +
+ "for uncaching in {} ",
+ key, DurationFormatUtils.formatDurationHMS(revocationPollingMs));
+ }
deferredUncachingExecutor.schedule(
new UncachingTask(key, revocationMs),
revocationPollingMs, TimeUnit.MILLISECONDS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
index 12733fd..b760dd8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
@@ -320,9 +320,11 @@ public class FileJournalManager implements JournalManager {
Collection<EditLogInputStream> streams, long fromTxId,
boolean inProgressOk) throws IOException {
List<EditLogFile> elfs = matchEditLogs(sd.getCurrentDir());
- LOG.debug(this + ": selecting input streams starting at " + fromTxId +
- (inProgressOk ? " (inProgress ok) " : " (excluding inProgress) ") +
- "from among " + elfs.size() + " candidate file(s)");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(this + ": selecting input streams starting at " + fromTxId +
+ (inProgressOk ? " (inProgress ok) " : " (excluding inProgress) ") +
+ "from among " + elfs.size() + " candidate file(s)");
+ }
addStreamsToCollectionFromFiles(elfs, streams, fromTxId, inProgressOk);
}
@@ -331,8 +333,10 @@ public class FileJournalManager implements JournalManager {
for (EditLogFile elf : elfs) {
if (elf.isInProgress()) {
if (!inProgressOk) {
- LOG.debug("passing over " + elf + " because it is in progress " +
- "and we are ignoring in-progress logs.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("passing over " + elf + " because it is in progress " +
+ "and we are ignoring in-progress logs.");
+ }
continue;
}
try {
@@ -345,9 +349,11 @@ public class FileJournalManager implements JournalManager {
}
if (elf.lastTxId < fromTxId) {
assert elf.lastTxId != HdfsConstants.INVALID_TXID;
- LOG.debug("passing over " + elf + " because it ends at " +
- elf.lastTxId + ", but we only care about transactions " +
- "as new as " + fromTxId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("passing over " + elf + " because it ends at " +
+ elf.lastTxId + ", but we only care about transactions " +
+ "as new as " + fromTxId);
+ }
continue;
}
EditLogFileInputStream elfis = new EditLogFileInputStream(elf.getFile(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 04be1ad..4575d64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -1534,7 +1534,9 @@ public class NameNode implements NameNodeStatusMXBean {
URI defaultUri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://"
+ conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY));
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
- LOG.debug("Setting " + FS_DEFAULT_NAME_KEY + " to " + defaultUri.toString());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting " + FS_DEFAULT_NAME_KEY + " to " + defaultUri.toString());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index bccfd6e..73c52d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -446,7 +446,9 @@ public class ShortCircuitCache implements Closeable {
purgeReason = "purging replica because it is stale.";
}
if (purgeReason != null) {
- LOG.debug(this + ": " + purgeReason);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(this + ": " + purgeReason);
+ }
purge(replica);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index fd29106..351ff03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -147,8 +147,10 @@ class FSImageLoader {
summary.getCodec(), new BufferedInputStream(new LimitInputStream(
fin, s.getLength())));
- LOG.debug("Loading section " + s.getName() + " length: " + s.getLength
- ());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading section " + s.getName() + " length: " + s.getLength
+ ());
+ }
switch (FSImageFormatProtobuf.SectionName.fromString(s.getName())) {
case STRING_TABLE:
stringTable = loadStringTable(is);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java
index b60ef70..55655ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightHashSet.java
@@ -126,8 +126,10 @@ public class LightWeightHashSet<T> implements Collection<T> {
this.shrinkThreshold = (int) (capacity * minLoadFactor);
entries = new LinkedElement[capacity];
- LOG.debug("initial capacity=" + initialCapacity + ", max load factor= "
- + maxLoadFactor + ", min load factor= " + minLoadFactor);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("initial capacity=" + initialCapacity + ", max load factor= "
+ + maxLoadFactor + ", min load factor= " + minLoadFactor);
+ }
}
public LightWeightHashSet() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java
index fc18dbe..164692e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java
@@ -136,7 +136,9 @@ final class TokenAspect<T extends FileSystem & Renewable> {
if (token != null) {
fs.setDelegationToken(token);
addRenewAction(fs);
- LOG.debug("Created new DT for " + token.getService());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Created new DT for " + token.getService());
+ }
}
hasInitedToken = true;
}
@@ -149,7 +151,9 @@ final class TokenAspect<T extends FileSystem & Renewable> {
synchronized void initDelegationToken(UserGroupInformation ugi) {
Token<?> token = selectDelegationToken(ugi);
if (token != null) {
- LOG.debug("Found existing DT for " + token.getService());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Found existing DT for " + token.getService());
+ }
fs.setDelegationToken(token);
hasInitedToken = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9fbc73e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 12adb05..6a55899 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -224,12 +224,16 @@ public class WebHdfsFileSystem extends FileSystem
// refetch tokens. even if ugi has credentials, don't attempt
// to get another token to match hdfs/rpc behavior
if (token != null) {
- LOG.debug("Using UGI token: " + token);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Using UGI token: " + token);
+ }
canRefreshDelegationToken = false;
} else {
token = getDelegationToken(null);
if (token != null) {
- LOG.debug("Fetched new token: " + token);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Fetched new token: " + token);
+ }
} else { // security is disabled
canRefreshDelegationToken = false;
}
@@ -244,7 +248,9 @@ public class WebHdfsFileSystem extends FileSystem
boolean replaced = false;
if (canRefreshDelegationToken) {
Token<?> token = getDelegationToken(null);
- LOG.debug("Replaced expired token: " + token);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Replaced expired token: " + token);
+ }
setDelegationToken(token);
replaced = (token != null);
}
@@ -1188,7 +1194,9 @@ public class WebHdfsFileSystem extends FileSystem
cancelDelegationToken(delegationToken);
}
} catch (IOException ioe) {
- LOG.debug("Token cancel failed: "+ioe);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Token cancel failed: " + ioe);
+ }
} finally {
super.close();
}
[34/50] [abbrv] hadoop git commit: HADOOP-11797. releasedocmaker.py
needs to put ASF headers on output (aw)
Posted by zj...@apache.org.
HADOOP-11797. releasedocmaker.py needs to put ASF headers on output (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c134f991
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c134f991
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c134f991
Branch: refs/heads/YARN-2928
Commit: c134f991272f40c87ddeeafb07fb96e801ec53df
Parents: d5d8402
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Apr 2 20:50:36 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:15 2015 -0700
----------------------------------------------------------------------
dev-support/releasedocmaker.py | 23 ++++++++++++++++++++
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c134f991/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index b00c1a7..78c8e6b 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -30,6 +30,26 @@ except ImportError:
releaseVersion={}
namePattern = re.compile(r' \([0-9]+\)')
+asflicense='''
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+'''
+
def clean(str):
return tableclean(re.sub(namePattern, "", str))
@@ -357,6 +377,9 @@ def main():
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
+ reloutputs.writeAll(asflicense)
+ choutputs.writeAll(asflicense)
+
relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c134f991/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 260cc83..495c42d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -431,6 +431,8 @@ Trunk (Unreleased)
HADOOP-11703. git should ignore .DS_Store files on Mac OS X (Abin Shahab
via aw)
+ HADOOP-11797. releasedocmaker.py needs to put ASF headers on output (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
[43/50] [abbrv] hadoop git commit: HADOOP-11377. Added Null.java
without which jdiff completely flops. Contributed by Tsuyoshi Ozawa.
Posted by zj...@apache.org.
HADOOP-11377. Added Null.java without which jdiff completely flops. Contributed by Tsuyoshi Ozawa.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3485a8d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3485a8d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3485a8d7
Branch: refs/heads/YARN-2928
Commit: 3485a8d7b42817ddbf4b13c68bf9c195806c1afc
Parents: e1ffb3e
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Sat Apr 4 13:47:08 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:16 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/dev-support/jdiff/Null.java | 20 ++++++++++++++++++++
.../hadoop-hdfs/dev-support/jdiff/Null.java | 20 ++++++++++++++++++++
3 files changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3485a8d7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d2d1181..fd926aa 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1202,6 +1202,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11757. NFS gateway should shutdown when it can't start UDP or TCP
server (brandonli)
+
+ HADOOP-11377. Added Null.java without which jdiff completely flops. (Tsuyoshi
+ Ozawa via vinodkv)
Release 2.6.1 - UNRELEASED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3485a8d7/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java b/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java
new file mode 100644
index 0000000..7b00145
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Null.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class Null {
+ public Null() { }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3485a8d7/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java
new file mode 100644
index 0000000..7b00145
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Null.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class Null {
+ public Null() { }
+}
[44/50] [abbrv] hadoop git commit: HADOOP-11776. Fixed the broken
JDiff support in Hadoop 2. Contributed by Li Lu.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 33f5a04..d206593 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -165,8 +165,10 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <jdiff.stable.api>0.20.2</jdiff.stable.api>
+ <jdiff.stable.api>2.6.0</jdiff.stable.api>
<jdiff.stability>-unstable</jdiff.stability>
+ <!-- Commented out for HADOOP-11776 -->
+ <!-- Uncomment param name="${jdiff.compatibility}" in javadoc doclet if compatibility is not empty -->
<jdiff.compatibility></jdiff.compatibility>
<jdiff.javadoc.maxmemory>512m</jdiff.javadoc.maxmemory>
</properties>
@@ -227,6 +229,14 @@
<outputDirectory>${project.build.directory}</outputDirectory>
<destFileName>hadoop-annotations.jar</destFileName>
</artifactItem>
+ <artifactItem>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>${xerces.jdiff.version}</version>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <destFileName>xerces.jar</destFileName>
+ </artifactItem>
</artifactItems>
</configuration>
</execution>
@@ -276,7 +286,7 @@
<doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
path="${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar">
<param name="-apidir" value="${project.build.directory}/site/jdiff/xml"/>
- <param name="-apiname" value="hadoop-core ${project.version}"/>
+ <param name="-apiname" value="${project.name} ${project.version}"/>
<param name="${jdiff.stability}"/>
</doclet>
<packageset dir="${basedir}/src/main/java"/>
@@ -289,17 +299,17 @@
sourceFiles="${basedir}/dev-support/jdiff/Null.java"
maxmemory="${jdiff.javadoc.maxmemory}">
<doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
- path="${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar">
- <param name="-oldapi" value="hadoop-core ${jdiff.stable.api}"/>
- <param name="-newapi" value="hadoop-core ${project.version}"/>
+ path="${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar:${project.build.directory}/xerces.jar">
+ <param name="-oldapi" value="${project.name} ${jdiff.stable.api}"/>
+ <param name="-newapi" value="${project.name} ${project.version}"/>
<param name="-oldapidir" value="${basedir}/dev-support/jdiff"/>
<param name="-newapidir" value="${project.build.directory}/site/jdiff/xml"/>
<param name="-javadocold"
- value="http://hadoop.apache.org/docs/${jdiff.stable.api}/api/"/>
+ value="http://hadoop.apache.org/docs/r${jdiff.stable.api}/api/"/>
<param name="-javadocnew" value="${project.build.directory}/site/api"/>
<param name="-stats"/>
<param name="${jdiff.stability}"/>
- <param name="${jdiff.compatibility}"/>
+ <!--param name="${jdiff.compatibility}"/-->
</doclet>
<classpath>
<path refid="maven.compile.classpath"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1069dcee/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2444c23..a2def1a 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -42,6 +42,8 @@
<!-- These 2 versions are defined here becuase they are used -->
<!-- JDIFF generation from embedded ant in the antrun plugin -->
<jdiff.version>1.0.9</jdiff.version>
+ <!-- Version number for xerces used by JDiff -->
+ <xerces.jdiff.version>2.11.0</xerces.jdiff.version>
<hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
[26/50] [abbrv] hadoop git commit: HDFS-7888. Change DFSOutputStream
and DataStreamer for convenience of subclassing. Contributed by Li Bo
Posted by zj...@apache.org.
HDFS-7888. Change DFSOutputStream and DataStreamer for convenience of subclassing. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b98f637c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b98f637c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b98f637c
Branch: refs/heads/YARN-2928
Commit: b98f637c1843b1f79016fd86244ecc68039ba54a
Parents: b4c3ce1
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Apr 2 10:59:26 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 6 12:08:13 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 116 ++++++++++---------
.../org/apache/hadoop/hdfs/DataStreamer.java | 13 +--
3 files changed, 69 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b98f637c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 80d958d..0c66309 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -376,6 +376,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7978. Add LOG.isDebugEnabled() guard for some LOG.debug(..).
(Walter Su via wang)
+ HDFS-7888. Change DFSOutputStream and DataStreamer for convenience of
+ subclassing. (Li Bo via szetszwo)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b98f637c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index c88639d..f6733e3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
-import java.net.Socket;
import java.nio.channels.ClosedChannelException;
import java.util.EnumSet;
import java.util.concurrent.atomic.AtomicReference;
@@ -95,29 +94,29 @@ public class DFSOutputStream extends FSOutputSummer
static CryptoProtocolVersion[] SUPPORTED_CRYPTO_VERSIONS =
CryptoProtocolVersion.supported();
- private final DFSClient dfsClient;
- private final ByteArrayManager byteArrayManager;
+ protected final DFSClient dfsClient;
+ protected final ByteArrayManager byteArrayManager;
// closed is accessed by different threads under different locks.
- private volatile boolean closed = false;
-
- private final String src;
- private final long fileId;
- private final long blockSize;
- private final int bytesPerChecksum;
-
- private DFSPacket currentPacket = null;
- private DataStreamer streamer;
- private int packetSize = 0; // write packet size, not including the header.
- private int chunksPerPacket = 0;
- private long lastFlushOffset = 0; // offset when flush was invoked
+ protected volatile boolean closed = false;
+
+ protected final String src;
+ protected final long fileId;
+ protected final long blockSize;
+ protected final int bytesPerChecksum;
+
+ protected DFSPacket currentPacket = null;
+ protected DataStreamer streamer;
+ protected int packetSize = 0; // write packet size, not including the header.
+ protected int chunksPerPacket = 0;
+ protected long lastFlushOffset = 0; // offset when flush was invoked
private long initialFileSize = 0; // at time of file open
private final short blockReplication; // replication factor of file
- private boolean shouldSyncBlock = false; // force blocks to disk upon close
- private final AtomicReference<CachingStrategy> cachingStrategy;
+ protected boolean shouldSyncBlock = false; // force blocks to disk upon close
+ protected final AtomicReference<CachingStrategy> cachingStrategy;
private FileEncryptionInfo fileEncryptionInfo;
/** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/
- private DFSPacket createPacket(int packetSize, int chunksPerPkt, long offsetInBlock,
+ protected DFSPacket createPacket(int packetSize, int chunksPerPkt, long offsetInBlock,
long seqno, boolean lastPacketInBlock) throws InterruptedIOException {
final byte[] buf;
final int bufferSize = PacketHeader.PKT_MAX_HEADER_LEN + packetSize;
@@ -206,7 +205,7 @@ public class DFSOutputStream extends FSOutputSummer
}
/** Construct a new output stream for creating a file. */
- private DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat,
+ protected DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat,
EnumSet<CreateFlag> flag, Progressable progress,
DataChecksum checksum, String[] favoredNodes) throws IOException {
this(dfsClient, src, progress, stat, checksum);
@@ -359,7 +358,7 @@ public class DFSOutputStream extends FSOutputSummer
}
}
- private void computePacketChunkSize(int psize, int csize) {
+ protected void computePacketChunkSize(int psize, int csize) {
final int bodySize = psize - PacketHeader.PKT_MAX_HEADER_LEN;
final int chunkSize = csize + getChecksumSize();
chunksPerPacket = Math.max(bodySize/chunkSize, 1);
@@ -426,33 +425,46 @@ public class DFSOutputStream extends FSOutputSummer
streamer.waitAndQueuePacket(currentPacket);
currentPacket = null;
- // If the reopened file did not end at chunk boundary and the above
- // write filled up its partial chunk. Tell the summer to generate full
- // crc chunks from now on.
- if (streamer.getAppendChunk() &&
- streamer.getBytesCurBlock() % bytesPerChecksum == 0) {
- streamer.setAppendChunk(false);
- resetChecksumBufSize();
- }
+ adjustChunkBoundary();
- if (!streamer.getAppendChunk()) {
- int psize = Math.min((int)(blockSize-streamer.getBytesCurBlock()),
- dfsClient.getConf().writePacketSize);
- computePacketChunkSize(psize, bytesPerChecksum);
- }
- //
- // if encountering a block boundary, send an empty packet to
- // indicate the end of block and reset bytesCurBlock.
- //
- if (streamer.getBytesCurBlock() == blockSize) {
- currentPacket = createPacket(0, 0, streamer.getBytesCurBlock(),
- streamer.getAndIncCurrentSeqno(), true);
- currentPacket.setSyncBlock(shouldSyncBlock);
- streamer.waitAndQueuePacket(currentPacket);
- currentPacket = null;
- streamer.setBytesCurBlock(0);
- lastFlushOffset = 0;
- }
+ endBlock();
+ }
+ }
+
+ /**
+ * If the reopened file did not end at chunk boundary and the above
+ * write filled up its partial chunk. Tell the summer to generate full
+ * crc chunks from now on.
+ */
+ protected void adjustChunkBoundary() {
+ if (streamer.getAppendChunk() &&
+ streamer.getBytesCurBlock() % bytesPerChecksum == 0) {
+ streamer.setAppendChunk(false);
+ resetChecksumBufSize();
+ }
+
+ if (!streamer.getAppendChunk()) {
+ int psize = Math.min((int)(blockSize- streamer.getBytesCurBlock()),
+ dfsClient.getConf().writePacketSize);
+ computePacketChunkSize(psize, bytesPerChecksum);
+ }
+ }
+
+ /**
+ * if encountering a block boundary, send an empty packet to
+ * indicate the end of block and reset bytesCurBlock.
+ *
+ * @throws IOException
+ */
+ protected void endBlock() throws IOException {
+ if (streamer.getBytesCurBlock() == blockSize) {
+ currentPacket = createPacket(0, 0, streamer.getBytesCurBlock(),
+ streamer.getAndIncCurrentSeqno(), true);
+ currentPacket.setSyncBlock(shouldSyncBlock);
+ streamer.waitAndQueuePacket(currentPacket);
+ currentPacket = null;
+ streamer.setBytesCurBlock(0);
+ lastFlushOffset = 0;
}
}
@@ -676,7 +688,7 @@ public class DFSOutputStream extends FSOutputSummer
* Waits till all existing data is flushed and confirmations
* received from datanodes.
*/
- private void flushInternal() throws IOException {
+ protected void flushInternal() throws IOException {
long toWaitFor;
synchronized (this) {
dfsClient.checkOpen();
@@ -692,7 +704,7 @@ public class DFSOutputStream extends FSOutputSummer
streamer.waitForAckedSeqno(toWaitFor);
}
- private synchronized void start() {
+ protected synchronized void start() {
streamer.start();
}
@@ -721,7 +733,7 @@ public class DFSOutputStream extends FSOutputSummer
// shutdown datastreamer and responseprocessor threads.
// interrupt datastreamer if force is true
- private void closeThreads(boolean force) throws IOException {
+ protected void closeThreads(boolean force) throws IOException {
try {
streamer.close(force);
streamer.join();
@@ -749,7 +761,7 @@ public class DFSOutputStream extends FSOutputSummer
}
}
- private synchronized void closeImpl() throws IOException {
+ protected synchronized void closeImpl() throws IOException {
if (isClosed()) {
IOException e = streamer.getLastException().getAndSet(null);
if (e == null)
@@ -761,7 +773,7 @@ public class DFSOutputStream extends FSOutputSummer
try {
flushBuffer(); // flush from all upper layers
- if (currentPacket != null) {
+ if (currentPacket != null) {
streamer.waitAndQueuePacket(currentPacket);
currentPacket = null;
}
@@ -792,7 +804,7 @@ public class DFSOutputStream extends FSOutputSummer
// should be called holding (this) lock since setTestFilename() may
// be called during unit tests
- private void completeFile(ExtendedBlock last) throws IOException {
+ protected void completeFile(ExtendedBlock last) throws IOException {
long localstart = Time.monotonicNow();
long sleeptime = dfsClient.getConf().
blockWriteLocateFollowingInitialDelayMs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b98f637c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 6ff4c24..6bcbfde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -1519,7 +1519,7 @@ class DataStreamer extends Daemon {
}
}
- private LocatedBlock locateFollowingBlock(DatanodeInfo[] excludedNodes)
+ protected LocatedBlock locateFollowingBlock(DatanodeInfo[] excludedNodes)
throws IOException {
int retries = dfsClient.getConf().nBlockWriteLocateFollowingRetry;
long sleeptime = dfsClient.getConf().
@@ -1729,15 +1729,6 @@ class DataStreamer extends Daemon {
}
/**
- * get the socket connecting to the first datanode in pipeline
- *
- * @return socket connecting to the first datanode in pipeline
- */
- Socket getSocket() {
- return s;
- }
-
- /**
* set socket to null
*/
void setSocketToNull() {
@@ -1814,4 +1805,4 @@ class DataStreamer extends Daemon {
s.close();
}
}
-}
\ No newline at end of file
+}