You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2016/01/25 18:25:36 UTC
[01/50] [abbrv] hadoop git commit: HADOOP-12356. Fix computing CPU
usage statistics on Windows. (Inigo Goiri via wangda)
Repository: hadoop
Updated Branches:
refs/heads/YARN-1011 96d8f1d6f -> 6eacdea0e
HADOOP-12356. Fix computing CPU usage statistics on Windows. (Inigo Goiri via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89d1fd5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89d1fd5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89d1fd5d
Branch: refs/heads/YARN-1011
Commit: 89d1fd5dac4bccf42d82686e146b02eb60d14736
Parents: edc43a9
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Jan 19 21:26:38 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Jan 19 21:27:38 2016 +0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/util/SysInfo.java | 11 ++++-
.../org/apache/hadoop/util/SysInfoLinux.java | 15 +++++-
.../org/apache/hadoop/util/SysInfoWindows.java | 26 ++++++++--
.../apache/hadoop/util/TestSysInfoLinux.java | 18 +++++--
.../apache/hadoop/util/TestSysInfoWindows.java | 51 ++++++++++++++++++--
.../gridmix/DummyResourceCalculatorPlugin.java | 2 +-
.../yarn/util/ResourceCalculatorPlugin.java | 15 ++++--
.../nodemanager/NodeResourceMonitorImpl.java | 4 +-
.../monitor/MockResourceCalculatorPlugin.java | 2 +-
.../util/TestNodeManagerHardwareUtils.java | 2 +-
11 files changed, 124 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b13876a..bd6550c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1622,6 +1622,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12700. Remove unused import in TestCompressorDecompressor.java.
(John Zhuge via aajisaka)
+ HADOOP-12356. Fix computing CPU usage statistics on Windows.
+ (Inigo Goiri via wangda)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
index b75a8d8..e8a5714 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
@@ -104,9 +104,16 @@ public abstract class SysInfo {
/**
* Obtain the CPU usage % of the machine. Return -1 if it is unavailable
*
- * @return CPU usage as a percentage of available cycles.
+ * @return CPU usage as a percentage (from 0 to 100) of available cycles.
*/
- public abstract float getCpuUsage();
+ public abstract float getCpuUsagePercentage();
+
+ /**
+ * Obtain the number of VCores used. Return -1 if it is unavailable
+ *
+ * @return Number of VCores used a percentage (from 0 to #VCores).
+ */
+ public abstract float getNumVCoresUsed();
/**
* Obtain the aggregated number of bytes read over the network.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 6b21f18..2852bea 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -608,7 +608,7 @@ public class SysInfoLinux extends SysInfo {
/** {@inheritDoc} */
@Override
- public float getCpuUsage() {
+ public float getCpuUsagePercentage() {
readProcStatFile();
float overallCpuUsage = cpuTimeTracker.getCpuTrackerUsagePercent();
if (overallCpuUsage != CpuTimeTracker.UNAVAILABLE) {
@@ -619,6 +619,17 @@ public class SysInfoLinux extends SysInfo {
/** {@inheritDoc} */
@Override
+ public float getNumVCoresUsed() {
+ readProcStatFile();
+ float overallVCoresUsage = cpuTimeTracker.getCpuTrackerUsagePercent();
+ if (overallVCoresUsage != CpuTimeTracker.UNAVAILABLE) {
+ overallVCoresUsage = overallVCoresUsage / 100F;
+ }
+ return overallVCoresUsage;
+ }
+
+ /** {@inheritDoc} */
+ @Override
public long getNetworkBytesRead() {
readProcNetInfoFile();
return numNetBytesRead;
@@ -676,7 +687,7 @@ public class SysInfoLinux extends SysInfo {
} catch (InterruptedException e) {
// do nothing
}
- System.out.println("CPU usage % : " + plugin.getCpuUsage());
+ System.out.println("CPU usage % : " + plugin.getCpuUsagePercentage());
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
index 3b009ef..b65569b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
@@ -104,8 +104,13 @@ public class SysInfoWindows extends SysInfo {
cpuFrequencyKhz = Long.parseLong(sysInfo[5]);
cumulativeCpuTimeMs = Long.parseLong(sysInfo[6]);
if (lastCumCpuTimeMs != -1) {
+ /**
+ * This number will be the aggregated usage across all cores in
+ * [0.0, 100.0]. For example, it will be 400.0 if there are 8
+ * cores and each of them is running at 50% utilization.
+ */
cpuUsage = (cumulativeCpuTimeMs - lastCumCpuTimeMs)
- / (refreshInterval * 1.0f);
+ * 100F / refreshInterval;
}
} catch (NumberFormatException nfe) {
LOG.warn("Error parsing sysInfo", nfe);
@@ -175,9 +180,24 @@ public class SysInfoWindows extends SysInfo {
/** {@inheritDoc} */
@Override
- public float getCpuUsage() {
+ public float getCpuUsagePercentage() {
refreshIfNeeded();
- return cpuUsage;
+ float ret = cpuUsage;
+ if (ret != -1) {
+ ret = ret / numProcessors;
+ }
+ return ret;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public float getNumVCoresUsed() {
+ refreshIfNeeded();
+ float ret = cpuUsage;
+ if (ret != -1) {
+ ret = ret / 100F;
+ }
+ return ret;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index 83f2e86..47fef33 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -229,7 +229,10 @@ public class TestSysInfoLinux {
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCumulativeCpuTime(),
FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
- assertEquals(plugin.getCpuUsage(), (float)(CpuTimeTracker.UNAVAILABLE),0.0);
+ assertEquals(plugin.getCpuUsagePercentage(),
+ (float)(CpuTimeTracker.UNAVAILABLE),0.0);
+ assertEquals(plugin.getNumVCoresUsed(),
+ (float)(CpuTimeTracker.UNAVAILABLE),0.0);
// Advance the time and sample again to test the CPU usage calculation
uTime += 100L;
@@ -237,13 +240,15 @@ public class TestSysInfoLinux {
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCumulativeCpuTime(),
FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
- assertEquals(plugin.getCpuUsage(), 6.25F, 0.0);
+ assertEquals(plugin.getCpuUsagePercentage(), 6.25F, 0.0);
+ assertEquals(plugin.getNumVCoresUsed(), 0.5F, 0.0);
- // Advance the time and sample again. This time, we call getCpuUsage() only.
+ // Advance the time and sample again. This time, we call getCpuUsagePercentage() only.
uTime += 600L;
plugin.advanceTime(300L);
updateStatFile(uTime, nTime, sTime);
- assertEquals(plugin.getCpuUsage(), 25F, 0.0);
+ assertEquals(plugin.getCpuUsagePercentage(), 25F, 0.0);
+ assertEquals(plugin.getNumVCoresUsed(), 2F, 0.0);
// Advance very short period of time (one jiffy length).
// In this case, CPU usage should not be updated.
@@ -252,7 +257,10 @@ public class TestSysInfoLinux {
updateStatFile(uTime, nTime, sTime);
assertEquals(plugin.getCumulativeCpuTime(),
FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
- assertEquals(plugin.getCpuUsage(), 25F, 0.0); // CPU usage is not updated.
+ assertEquals(
+ plugin.getCpuUsagePercentage(), 25F, 0.0); // CPU usage is not updated.
+ assertEquals(
+ plugin.getNumVCoresUsed(), 2F, 0.0); // CPU usage is not updated.
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoWindows.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoWindows.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoWindows.java
index 7924c02..2544e7c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoWindows.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoWindows.java
@@ -58,7 +58,10 @@ public class TestSysInfoWindows {
assertEquals(2805000L, tester.getCpuFrequency());
assertEquals(6261812L, tester.getCumulativeCpuTime());
// undef on first call
- assertEquals(-1.0, tester.getCpuUsage(), 0.0);
+ assertEquals((float)CpuTimeTracker.UNAVAILABLE,
+ tester.getCpuUsagePercentage(), 0.0);
+ assertEquals((float)CpuTimeTracker.UNAVAILABLE,
+ tester.getNumVCoresUsed(), 0.0);
}
@Test(timeout = 10000)
@@ -70,22 +73,60 @@ public class TestSysInfoWindows {
tester.getAvailablePhysicalMemorySize();
// verify information has been refreshed
assertEquals(6400417792L, tester.getAvailablePhysicalMemorySize());
- assertEquals(-1.0, tester.getCpuUsage(), 0.0);
+ assertEquals((float)CpuTimeTracker.UNAVAILABLE,
+ tester.getCpuUsagePercentage(), 0.0);
+ assertEquals((float)CpuTimeTracker.UNAVAILABLE,
+ tester.getNumVCoresUsed(), 0.0);
tester.setSysinfoString(
"17177038848,8589467648,15232745472,5400417792,1,2805000,6263012\r\n");
tester.getAvailablePhysicalMemorySize();
// verify information has not been refreshed
assertEquals(6400417792L, tester.getAvailablePhysicalMemorySize());
- assertEquals(-1.0, tester.getCpuUsage(), 0.0);
+ assertEquals((float)CpuTimeTracker.UNAVAILABLE,
+ tester.getCpuUsagePercentage(), 0.0);
+ assertEquals((float)CpuTimeTracker.UNAVAILABLE,
+ tester.getNumVCoresUsed(), 0.0);
// advance clock
tester.advance(SysInfoWindows.REFRESH_INTERVAL_MS + 1);
// verify information has been refreshed
assertEquals(5400417792L, tester.getAvailablePhysicalMemorySize());
- assertEquals((6263012 - 6261812) / (SysInfoWindows.REFRESH_INTERVAL_MS + 1f),
- tester.getCpuUsage(), 0.0);
+ assertEquals((6263012 - 6261812) * 100F /
+ (SysInfoWindows.REFRESH_INTERVAL_MS + 1f) / 1,
+ tester.getCpuUsagePercentage(), 0.0);
+ assertEquals((6263012 - 6261812) /
+ (SysInfoWindows.REFRESH_INTERVAL_MS + 1f) / 1,
+ tester.getNumVCoresUsed(), 0.0);
+ }
+
+ @Test(timeout = 10000)
+ public void refreshAndCpuUsageMulticore() throws InterruptedException {
+ // test with 12 cores
+ SysInfoWindowsMock tester = new SysInfoWindowsMock();
+ tester.setSysinfoString(
+ "17177038848,8589467648,15232745472,6400417792,12,2805000,6261812\r\n");
+ // verify information has been refreshed
+ assertEquals(6400417792L, tester.getAvailablePhysicalMemorySize());
+
+ tester.setSysinfoString(
+ "17177038848,8589467648,15232745472,5400417792,12,2805000,6263012\r\n");
+ // verify information has not been refreshed
+ assertEquals(6400417792L, tester.getAvailablePhysicalMemorySize());
+
+ // advance clock
+ tester.advance(SysInfoWindows.REFRESH_INTERVAL_MS + 1);
+
+ // verify information has been refreshed
+ assertEquals(5400417792L, tester.getAvailablePhysicalMemorySize());
+ // verify information has been refreshed
+ assertEquals((6263012 - 6261812) * 100F /
+ (SysInfoWindows.REFRESH_INTERVAL_MS + 1f) / 12,
+ tester.getCpuUsagePercentage(), 0.0);
+ assertEquals((6263012 - 6261812) /
+ (SysInfoWindows.REFRESH_INTERVAL_MS + 1f),
+ tester.getNumVCoresUsed(), 0.0);
}
@Test(timeout = 10000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
index 4999f14..01d9242 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
@@ -120,7 +120,7 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
/** {@inheritDoc} */
@Override
- public float getCpuUsage() {
+ public float getCpuUsagePercentage() {
return getConf().getFloat(CPU_USAGE, -1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
index 691c4ac..7b2ea56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
@@ -120,12 +120,21 @@ public class ResourceCalculatorPlugin extends Configured {
}
/**
- * Obtain the CPU usage % of the machine. Return -1 if it is unavailable
+ * Obtain the CPU usage % of the machine. Return -1 if it is unavailable.
*
* @return CPU usage in %
*/
- public float getCpuUsage() {
- return sys.getCpuUsage();
+ public float getCpuUsagePercentage() {
+ return sys.getCpuUsagePercentage();
+ }
+
+ /**
+ * Obtain the number of VCores used. Return -1 if it is unavailable.
+ *
+ * @return Number of VCores used a percentage (from 0 to #VCores)
+ */
+ public float getNumVCoresUsed() {
+ return sys.getNumVCoresUsed();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index b161f14..3a78d87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -141,12 +141,12 @@ public class NodeResourceMonitorImpl extends AbstractService implements
long vmem =
resourceCalculatorPlugin.getVirtualMemorySize()
- resourceCalculatorPlugin.getAvailableVirtualMemorySize();
- float cpu = resourceCalculatorPlugin.getCpuUsage();
+ float vcores = resourceCalculatorPlugin.getNumVCoresUsed();
nodeUtilization =
ResourceUtilization.newInstance(
(int) (pmem >> 20), // B -> MB
(int) (vmem >> 20), // B -> MB
- cpu); // 1 CPU at 100% is 1
+ vcores); // Used Virtual Cores
try {
Thread.sleep(monitoringInterval);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
index 4a18a8c..bbde9ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
@@ -63,7 +63,7 @@ public class MockResourceCalculatorPlugin extends ResourceCalculatorPlugin {
}
@Override
- public float getCpuUsage() {
+ public float getCpuUsagePercentage() {
return 0;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89d1fd5d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
index 84a045d..4add586 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
@@ -73,7 +73,7 @@ public class TestNodeManagerHardwareUtils {
}
@Override
- public float getCpuUsage() {
+ public float getCpuUsagePercentage() {
return 0;
}
[24/50] [abbrv] hadoop git commit: HDFS-9674. The HTrace span for
OpWriteBlock should record the maxWriteToDisk time. Contributed by Colin
McCabe.
Posted by ka...@apache.org.
HDFS-9674. The HTrace span for OpWriteBlock should record the maxWriteToDisk time. Contributed by Colin McCabe.
Change-Id: I9bf3f3bcd57f5880189ad7c160f3dd66f97d904b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4a05c1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4a05c1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4a05c1f
Branch: refs/heads/YARN-1011
Commit: b4a05c1fd5ae98e28cf2b15f5b068e6c2d12f2a8
Parents: 2a86735
Author: Zhe Zhang <zh...@apache.org>
Authored: Thu Jan 21 13:25:42 2016 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Thu Jan 21 13:25:42 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/BlockReceiver.java | 11 +++++++++++
2 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4a05c1f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index da16357..0ded0c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -941,6 +941,9 @@ Release 2.9.0 - UNRELEASED
HDFS-9542. Move BlockIdManager from FSNamesystem to BlockManager. (jing9)
+ HDFS-9674. The HTrace span for OpWriteBlock should record the maxWriteToDisk
+ time. (cmccabe via zhz)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4a05c1f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 8003c76..e7908a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
+import org.apache.htrace.core.Span;
+import org.apache.htrace.core.Tracer;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.SYNC_FILE_RANGE_WRITE;
@@ -136,6 +138,7 @@ class BlockReceiver implements Closeable {
private long lastResponseTime = 0;
private boolean isReplaceBlock = false;
private DataOutputStream replyOut = null;
+ private long maxWriteToDiskMs = 0;
private boolean pinning;
private long lastSentTime;
@@ -302,6 +305,11 @@ class BlockReceiver implements Closeable {
*/
@Override
public void close() throws IOException {
+ Span span = Tracer.getCurrentSpan();
+ if (span != null) {
+ span.addKVAnnotation("maxWriteToDiskMs",
+ Long.toString(maxWriteToDiskMs));
+ }
packetReceiver.close();
IOException ioe = null;
@@ -697,6 +705,9 @@ class BlockReceiver implements Closeable {
long begin = Time.monotonicNow();
out.write(dataBuf.array(), startByteToDisk, numBytesToDisk);
long duration = Time.monotonicNow() - begin;
+ if (duration > maxWriteToDiskMs) {
+ maxWriteToDiskMs = duration;
+ }
if (duration > datanodeSlowLogThresholdMs) {
LOG.warn("Slow BlockReceiver write data to disk cost:" + duration
+ "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
[07/50] [abbrv] hadoop git commit: HDFS-9655. NN should start JVM
pause monitor before loading fsimage. (John Zhuge via Lei (Eddy) Xu)
Posted by ka...@apache.org.
HDFS-9655. NN should start JVM pause monitor before loading fsimage. (John Zhuge via Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ec438e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ec438e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ec438e8
Branch: refs/heads/YARN-1011
Commit: 2ec438e8f7cd77cb48fd1264781e60a48e331908
Parents: 7905788
Author: Lei Xu <le...@apache.org>
Authored: Wed Jan 20 14:26:11 2016 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Wed Jan 20 14:26:11 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/util/JvmPauseMonitor.java | 1 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/namenode/NameNode.java | 12 ++++++------
3 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ec438e8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index 882e4a7..17a1952 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -185,6 +185,7 @@ public class JvmPauseMonitor extends AbstractService {
public void run() {
StopWatch sw = new StopWatch();
Map<String, GcTimes> gcTimesBeforeSleep = getGcTimes();
+ LOG.info("Starting JVM pause monitor");
while (shouldRun) {
sw.reset().start();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ec438e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7a5290e..e6022fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1822,6 +1822,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9415. Document dfs.cluster.administrators and
dfs.permissions.superusergroup. (Xiaobing Zhou via Arpit Agarwal)
+ HDFS-9655. NN should start JVM pause monitor before loading fsimage.
+ (John Zhuge via Lei (Eddy) Xu)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ec438e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 97c8b26..7c4e10f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -671,6 +671,11 @@ public class NameNode implements NameNodeStatusMXBean {
NameNode.initMetrics(conf, this.getRole());
StartupProgressMetrics.register(startupProgress);
+ pauseMonitor = new JvmPauseMonitor();
+ pauseMonitor.init(conf);
+ pauseMonitor.start();
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
+
if (NamenodeRole.NAMENODE == role) {
startHttpServer(conf);
}
@@ -690,12 +695,7 @@ public class NameNode implements NameNodeStatusMXBean {
httpServer.setNameNodeAddress(getNameNodeAddress());
httpServer.setFSImage(getFSImage());
}
-
- pauseMonitor = new JvmPauseMonitor();
- pauseMonitor.init(conf);
- pauseMonitor.start();
- metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
-
+
startCommonServices(conf);
startMetricsLogger(conf);
}
[10/50] [abbrv] hadoop git commit: YARN-4557. Fix improper Queues
sorting in PartitionedQueueComparator when accessible-node-labels=*.
(Naganarasimha G R via wangda)
Posted by ka...@apache.org.
YARN-4557. Fix improper Queues sorting in PartitionedQueueComparator when accessible-node-labels=*. (Naganarasimha G R via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ff5f673
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ff5f673
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ff5f673
Branch: refs/heads/YARN-1011
Commit: 5ff5f67332b527acaca7a69ac421930a02ca55b3
Parents: 1708a4c
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jan 21 11:15:04 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Jan 21 11:21:06 2016 +0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/AppSchedulingInfo.java | 2 +-
.../capacity/PartitionedQueueComparator.java | 10 ++-
.../TestNodeLabelContainerAllocation.java | 77 +++++++++++++++++++-
4 files changed, 87 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff5f673/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 212ffa1..bd467ea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1289,6 +1289,9 @@ Release 2.8.0 - UNRELEASED
YARN-4565. Fix a bug that leads to AM resource limit not hornored when
sizeBasedWeight enabled for FairOrderingPolicy. (wtan via jianhe)
+ YARN-4557. Fix improper Queues sorting in PartitionedQueueComparator
+ when accessible-node-labels=*. (Naganarasimha G R via wangda)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff5f673/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 631b418..07f3d8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class AppSchedulingInfo {
private static final Log LOG = LogFactory.getLog(AppSchedulingInfo.class);
- private static final Comparator COMPARATOR =
+ private static final Comparator<Priority> COMPARATOR =
new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator();
private static final int EPOCH_BIT_SHIFT = 40;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff5f673/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PartitionedQueueComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PartitionedQueueComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PartitionedQueueComparator.java
index ddcc719..477c615 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PartitionedQueueComparator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PartitionedQueueComparator.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.util.Comparator;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+
public class PartitionedQueueComparator implements Comparator<CSQueue> {
private String partitionToLookAt = null;
@@ -35,15 +37,17 @@ public class PartitionedQueueComparator implements Comparator<CSQueue> {
* the other not, accessible queue goes first.
*/
boolean q1Accessible =
- q1.getAccessibleNodeLabels().contains(partitionToLookAt);
+ q1.getAccessibleNodeLabels().contains(partitionToLookAt)
+ || q1.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY);
boolean q2Accessible =
- q2.getAccessibleNodeLabels().contains(partitionToLookAt);
+ q2.getAccessibleNodeLabels().contains(partitionToLookAt)
+ || q2.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY);
if (q1Accessible && !q2Accessible) {
return -1;
} else if (!q1Accessible && q2Accessible) {
return 1;
}
-
+
/*
*
* 2. When two queue has same accessibility, check who will go first:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ff5f673/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index dff82ca..bbf6e43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -502,7 +503,6 @@ public class TestNodeLabelContainerAllocation {
};
rm1.getRMContext().setNodeLabelManager(mgr);
rm1.start();
- MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // no label
MockNM nm2 = rm1.registerNode("h2:1234", 40 * GB); // label = y
// launch an app to queue b1 (label = y), AM container should be launched in
// nm2
@@ -1470,9 +1470,11 @@ public class TestNodeLabelContainerAllocation {
csConf.setCapacityByLabel(B, "x", 70);
final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+ csConf.setAccessibleNodeLabels(C, Collections.<String> emptySet());
csConf.setCapacity(C, 25);
final String D = CapacitySchedulerConfiguration.ROOT + ".d";
+ csConf.setAccessibleNodeLabels(D, Collections.<String> emptySet());
csConf.setCapacity(D, 25);
// set node -> label
@@ -1601,4 +1603,77 @@ public class TestNodeLabelContainerAllocation {
cs.getApplicationAttempt(am4.getApplicationAttemptId()));
}
+
+ @Test
+ public void testOrderOfAllocationOnPartitionsWhenAccessibilityIsAll()
+ throws Exception {
+ /**
+ * Test case: have a following queue structure:
+ *
+ * <pre>
+ * root
+ * __________
+ * / \
+ * a (*) b (x)
+ * </pre>
+ *
+ * Both queues a/b can access x, we need to verify whether * accessibility
+ * is considered in ordering of queues
+ */
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(this.conf);
+
+ // Define top-level queues
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] { "a", "b" });
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ csConf.setCapacity(A, 25);
+ csConf.setAccessibleNodeLabels(A, toSet("*"));
+ csConf.setCapacityByLabel(A, "x", 60);
+
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ csConf.setCapacity(B, 75);
+ csConf.setAccessibleNodeLabels(B, toSet("x"));
+ csConf.setCapacityByLabel(B, "x", 40);
+
+ // set node -> label
+ mgr.addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x", false)));
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+
+ // inject node label manager
+ MockRM rm = new MockRM(csConf) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm.getRMContext().setNodeLabelManager(mgr);
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ MockNM nm1 = rm.registerNode("h1:1234", 10 * GB); // label = x
+
+ // app1 -> a
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a", "x");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ // app2 -> b
+ RMApp app2 = rm.submitApp(1 * GB, "app", "user", null, "b", "x");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
+
+ // Both a/b has used_capacity(x) = 0, when doing exclusive allocation, a
+ // will go first since a has more capacity(x)
+ am1.allocate("*", 1 * GB, 1, new ArrayList<ContainerId>(), "x");
+ am2.allocate("*", 1 * GB, 1, new ArrayList<ContainerId>(), "x");
+ doNMHeartbeat(rm, nm1.getNodeId(), 1);
+ checkNumOfContainersInAnAppOnGivenNode(2, nm1.getNodeId(),
+ cs.getApplicationAttempt(am1.getApplicationAttemptId()));
+ }
}
[28/50] [abbrv] hadoop git commit: HDFS-9618. Fix mismatch between
log level and guard in BlockManager#computeRecoveryWorkForBlocks (iwasakims)
Posted by ka...@apache.org.
HDFS-9618. Fix mismatch between log level and guard in BlockManager#computeRecoveryWorkForBlocks (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae9c61ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae9c61ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae9c61ff
Branch: refs/heads/YARN-1011
Commit: ae9c61ff0a90b070a5b7b2e7160d726e92c8eacf
Parents: e306681
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Fri Jan 22 14:11:48 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Fri Jan 22 14:11:48 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/blockmanagement/BlockManager.java | 5 ++---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae9c61ff/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 140be77..8c26ee7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -951,6 +951,9 @@ Release 2.9.0 - UNRELEASED
HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle
NotReplicatedYetException on adding block (iwasakims)
+ HDFS-9618. Fix mismatch between log level and guard in
+ BlockManager#computeRecoveryWorkForBlocks (iwasakims)
+
HDFS-9621. getListing wrongly associates Erasure Coding policy to pre-existing
replicated files under an EC directory. (jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae9c61ff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index bf63708..d255471 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1488,7 +1488,7 @@ public class BlockManager implements BlockStatsMXBean {
namesystem.writeUnlock();
}
- if (blockLog.isInfoEnabled()) {
+ if (blockLog.isDebugEnabled()) {
// log which blocks have been scheduled for replication
for(BlockRecoveryWork rw : recovWork){
DatanodeStorageInfo[] targets = rw.getTargets();
@@ -1502,8 +1502,7 @@ public class BlockManager implements BlockStatsMXBean {
rw.getBlock(), targetList);
}
}
- }
- if (blockLog.isDebugEnabled()) {
+
blockLog.debug("BLOCK* neededReplications = {} pendingReplications = {}",
neededReplications.size(), pendingReplications.size());
}
[20/50] [abbrv] hadoop git commit: HDFS-9542. Move BlockIdManager
from FSNamesystem to BlockManager. Contributed by Jing Zhao.
Posted by ka...@apache.org.
HDFS-9542. Move BlockIdManager from FSNamesystem to BlockManager. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c304890c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c304890c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c304890c
Branch: refs/heads/YARN-1011
Commit: c304890c8c7782d835896859f5b7f60b96c306c0
Parents: 468a53b
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Jan 21 11:13:17 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Jan 21 11:13:17 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockIdManager.java | 19 +++---
.../server/blockmanagement/BlockManager.java | 66 ++++++++++----------
.../blockmanagement/BlockManagerSafeMode.java | 6 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 2 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 11 ++--
.../hdfs/server/namenode/FSImageFormat.java | 27 ++++----
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/FSNamesystem.java | 57 +++++++++--------
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 11 ----
.../blockmanagement/TestBlockManager.java | 2 +-
.../TestBlockManagerSafeMode.java | 6 +-
.../blockmanagement/TestReplicationPolicy.java | 6 +-
.../TestSequentialBlockGroupId.java | 12 ++--
.../blockmanagement/TestSequentialBlockId.java | 4 +-
.../hdfs/server/namenode/TestFileTruncate.java | 4 +-
.../hdfs/server/namenode/TestSaveNamespace.java | 4 +-
18 files changed, 120 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f41537e..da16357 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -939,6 +939,8 @@ Release 2.9.0 - UNRELEASED
HDFS-9576: HTrace: collect position/length information on read operations
(zhz via cmccabe)
+ HDFS-9542. Move BlockIdManager from FSNamesystem to BlockManager. (jing9)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
index 17fef05..9c71287 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -163,7 +163,7 @@ public class BlockIdManager {
/**
* Increments, logs and then returns the stamp
*/
- public long nextGenerationStamp(boolean legacyBlock) throws IOException {
+ long nextGenerationStamp(boolean legacyBlock) throws IOException {
return legacyBlock ? getNextGenerationStampV1() :
getNextGenerationStampV2();
}
@@ -199,22 +199,19 @@ public class BlockIdManager {
*
* @return true if the block ID was randomly generated, false otherwise.
*/
- public boolean isLegacyBlock(Block block) {
+ boolean isLegacyBlock(Block block) {
return block.getGenerationStamp() < getGenerationStampV1Limit();
}
/**
* Increments, logs and then returns the block ID
*/
- public long nextContiguousBlockId() {
- return blockIdGenerator.nextValue();
+ long nextBlockId(boolean isStriped) {
+ return isStriped ? blockGroupIdGenerator.nextValue() :
+ blockIdGenerator.nextValue();
}
- public long nextStripedBlockId() {
- return blockGroupIdGenerator.nextValue();
- }
-
- public boolean isGenStampInFuture(Block block) {
+ boolean isGenStampInFuture(Block block) {
if (isLegacyBlock(block)) {
return block.getGenerationStamp() > getGenerationStampV1();
} else {
@@ -222,7 +219,7 @@ public class BlockIdManager {
}
}
- public void clear() {
+ void clear() {
generationStampV1.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
generationStampV2.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
getBlockIdGenerator().setCurrentValue(SequentialBlockIdGenerator
@@ -240,7 +237,7 @@ public class BlockIdManager {
* and the other 60 bits are 1. Group ID is the first 60 bits of any
* data/parity block id in the same striped block group.
*/
- public static long convertToStripedID(long id) {
+ static long convertToStripedID(long id) {
return id & (~HdfsServerConstants.BLOCK_GROUP_INDEX_MASK);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index cee03f6..bf63708 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -78,7 +78,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
@@ -308,11 +307,14 @@ public class BlockManager implements BlockStatsMXBean {
/** Check whether there are any non-EC blocks using StripedID */
private boolean hasNonEcBlockUsingStripedID = false;
- public BlockManager(final Namesystem namesystem, final Configuration conf)
- throws IOException {
+ private final BlockIdManager blockIdManager;
+
+ public BlockManager(final Namesystem namesystem, boolean haEnabled,
+ final Configuration conf) throws IOException {
this.namesystem = namesystem;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
+ this.blockIdManager = new BlockIdManager(this);
startupDelayBlockDeletionInMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
@@ -387,7 +389,7 @@ public class BlockManager implements BlockStatsMXBean {
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
this.blockReportLeaseManager = new BlockReportLeaseManager(conf);
- bmSafeMode = new BlockManagerSafeMode(this, namesystem, conf);
+ bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
LOG.info("defaultReplication = " + defaultReplication);
LOG.info("maxReplication = " + maxReplication);
@@ -498,8 +500,7 @@ public class BlockManager implements BlockStatsMXBean {
/** Should the access keys be updated? */
boolean shouldUpdateBlockKey(final long updateTime) throws IOException {
- return isBlockTokenEnabled()? blockTokenSecretManager.updateKeys(updateTime)
- : false;
+ return isBlockTokenEnabled() && blockTokenSecretManager.updateKeys(updateTime);
}
public void activate(Configuration conf, long blockTotal) {
@@ -538,7 +539,7 @@ public class BlockManager implements BlockStatsMXBean {
/** Dump meta data to out. */
public void metaSave(PrintWriter out) {
- assert namesystem.hasWriteLock();
+ assert namesystem.hasWriteLock(); // TODO: block manager read lock and NS write lock
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
datanodeManager.fetchDatanodes(live, dead, false);
@@ -1108,27 +1109,8 @@ public class BlockManager implements BlockStatsMXBean {
return countNodes(b).liveReplicas() >= replication;
}
- /**
- * return a list of blocks & their locations on <code>datanode</code> whose
- * total size is <code>size</code>
- *
- * @param datanode on which blocks are located
- * @param size total size of blocks
- */
- public BlocksWithLocations getBlocks(DatanodeID datanode, long size
- ) throws IOException {
- namesystem.checkOperation(OperationCategory.READ);
- namesystem.readLock();
- try {
- namesystem.checkOperation(OperationCategory.READ);
- return getBlocksWithLocations(datanode, size);
- } finally {
- namesystem.readUnlock();
- }
- }
-
/** Get all blocks with location information from a datanode. */
- private BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
+ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
final long size) throws UnregisteredNodeException {
final DatanodeDescriptor node = getDatanodeManager().getDatanode(datanode);
if (node == null) {
@@ -2353,8 +2335,7 @@ public class BlockManager implements BlockStatsMXBean {
+ " on " + storageInfo.getDatanodeDescriptor() + " size " +
iblk.getNumBytes() + " replicaState = " + reportedState);
}
- if (shouldPostponeBlocksFromFuture &&
- namesystem.isGenStampInFuture(iblk)) {
+ if (shouldPostponeBlocksFromFuture && isGenStampInFuture(iblk)) {
queueReportedBlock(storageInfo, iblk, reportedState,
QUEUE_REASON_FUTURE_GENSTAMP);
continue;
@@ -2499,8 +2480,7 @@ public class BlockManager implements BlockStatsMXBean {
+ " replicaState = " + reportedState);
}
- if (shouldPostponeBlocksFromFuture &&
- namesystem.isGenStampInFuture(block)) {
+ if (shouldPostponeBlocksFromFuture && isGenStampInFuture(block)) {
queueReportedBlock(storageInfo, block, reportedState,
QUEUE_REASON_FUTURE_GENSTAMP);
return null;
@@ -3360,8 +3340,7 @@ public class BlockManager implements BlockStatsMXBean {
private void removeStoredBlock(DatanodeStorageInfo storageInfo, Block block,
DatanodeDescriptor node) {
- if (shouldPostponeBlocksFromFuture &&
- namesystem.isGenStampInFuture(block)) {
+ if (shouldPostponeBlocksFromFuture && isGenStampInFuture(block)) {
queueReportedBlock(storageInfo, block, null,
QUEUE_REASON_FUTURE_GENSTAMP);
return;
@@ -4201,6 +4180,7 @@ public class BlockManager implements BlockStatsMXBean {
}
public void clear() {
+ blockIdManager.clear();
clearQueues();
blocksMap.clear();
}
@@ -4364,4 +4344,24 @@ public class BlockManager implements BlockStatsMXBean {
}
}
}
+
+ public BlockIdManager getBlockIdManager() {
+ return blockIdManager;
+ }
+
+ public long nextGenerationStamp(boolean legacyBlock) throws IOException {
+ return blockIdManager.nextGenerationStamp(legacyBlock);
+ }
+
+ public boolean isLegacyBlock(Block block) {
+ return blockIdManager.isLegacyBlock(block);
+ }
+
+ public long nextBlockId(boolean isStriped) {
+ return blockIdManager.nextBlockId(isStriped);
+ }
+
+ boolean isGenStampInFuture(Block block) {
+ return blockIdManager.isGenStampInFuture(block);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index aba3c85..e2d688e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -115,10 +115,10 @@ class BlockManagerSafeMode {
private final boolean inRollBack;
BlockManagerSafeMode(BlockManager blockManager, Namesystem namesystem,
- Configuration conf) {
+ boolean haEnabled, Configuration conf) {
this.blockManager = blockManager;
this.namesystem = namesystem;
- this.haEnabled = namesystem.isHaEnabled();
+ this.haEnabled = haEnabled;
this.threshold = conf.getFloat(DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,
DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT);
if (this.threshold > 1.0) {
@@ -473,7 +473,7 @@ class BlockManagerSafeMode {
if (!blockManager.getShouldPostponeBlocksFromFuture() &&
!inRollBack &&
- namesystem.isGenStampInFuture(brr)) {
+ blockManager.isGenStampInFuture(brr)) {
numberOfBytesInFutureBlocks.addAndGet(brr.getBytesOnDisk());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
index 03eb96d..95980e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -228,7 +228,7 @@ final class FSDirTruncateOp {
if (newBlock == null) {
newBlock = (shouldCopyOnTruncate) ? fsn.createNewBlock(false)
: new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
- fsn.nextGenerationStamp(fsn.getBlockIdManager().isLegacyBlock(
+ fsn.nextGenerationStamp(fsn.getBlockManager().isLegacyBlock(
oldBlock)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index a74dd7f..5d27786 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
@@ -116,12 +117,14 @@ public class FSEditLogLoader {
static final long REPLAY_TRANSACTION_LOG_INTERVAL = 1000; // 1sec
private final FSNamesystem fsNamesys;
+ private final BlockManager blockManager;
private long lastAppliedTxId;
/** Total number of end transactions loaded. */
private int totalEdits = 0;
public FSEditLogLoader(FSNamesystem fsNamesys, long lastAppliedTxId) {
this.fsNamesys = fsNamesys;
+ this.blockManager = fsNamesys.getBlockManager();
this.lastAppliedTxId = lastAppliedTxId;
}
@@ -586,7 +589,7 @@ public class FSEditLogLoader {
}
case OP_SET_GENSTAMP_V1: {
SetGenstampV1Op setGenstampV1Op = (SetGenstampV1Op)op;
- fsNamesys.getBlockIdManager().setGenerationStampV1(
+ blockManager.getBlockIdManager().setGenerationStampV1(
setGenstampV1Op.genStampV1);
break;
}
@@ -794,7 +797,7 @@ public class FSEditLogLoader {
}
case OP_SET_GENSTAMP_V2: {
SetGenstampV2Op setGenstampV2Op = (SetGenstampV2Op) op;
- fsNamesys.getBlockIdManager().setGenerationStampV2(
+ blockManager.getBlockIdManager().setGenerationStampV2(
setGenstampV2Op.genStampV2);
break;
}
@@ -803,10 +806,10 @@ public class FSEditLogLoader {
if (BlockIdManager.isStripedBlockID(allocateBlockIdOp.blockId)) {
// ALLOCATE_BLOCK_ID is added for sequential block id, thus if the id
// is negative, it must belong to striped blocks
- fsNamesys.getBlockIdManager().setLastAllocatedStripedBlockId(
+ blockManager.getBlockIdManager().setLastAllocatedStripedBlockId(
allocateBlockIdOp.blockId);
} else {
- fsNamesys.getBlockIdManager().setLastAllocatedContiguousBlockId(
+ blockManager.getBlockIdManager().setLastAllocatedContiguousBlockId(
allocateBlockIdOp.blockId);
}
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 8c05a2d..a815864 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -48,11 +48,11 @@ import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutFlags;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -344,27 +344,26 @@ public class FSImageFormat {
// read in the last generation stamp for legacy blocks.
long genstamp = in.readLong();
- namesystem.getBlockIdManager().setGenerationStampV1(genstamp);
+ final BlockIdManager blockIdManager = namesystem.getBlockManager()
+ .getBlockIdManager();
+ blockIdManager.setGenerationStampV1(genstamp);
if (NameNodeLayoutVersion.supports(
LayoutVersion.Feature.SEQUENTIAL_BLOCK_ID, imgVersion)) {
// read the starting generation stamp for sequential block IDs
genstamp = in.readLong();
- namesystem.getBlockIdManager().setGenerationStampV2(genstamp);
+ blockIdManager.setGenerationStampV2(genstamp);
// read the last generation stamp for blocks created after
// the switch to sequential block IDs.
long stampAtIdSwitch = in.readLong();
- namesystem.getBlockIdManager().setGenerationStampV1Limit(stampAtIdSwitch);
+ blockIdManager.setGenerationStampV1Limit(stampAtIdSwitch);
// read the max sequential block ID.
long maxSequentialBlockId = in.readLong();
- namesystem.getBlockIdManager().setLastAllocatedContiguousBlockId(
- maxSequentialBlockId);
+ blockIdManager.setLastAllocatedContiguousBlockId(maxSequentialBlockId);
} else {
-
- long startingGenStamp = namesystem.getBlockIdManager()
- .upgradeGenerationStampToV2();
+ long startingGenStamp = blockIdManager.upgradeGenerationStampToV2();
// This is an upgrade.
LOG.info("Upgrading to sequential block IDs. Generation stamp " +
"for new blocks set to " + startingGenStamp);
@@ -1269,10 +1268,12 @@ public class FSImageFormat {
out.writeInt(sourceNamesystem.unprotectedGetNamespaceInfo()
.getNamespaceID());
out.writeLong(numINodes);
- out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampV1());
- out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampV2());
- out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampAtblockIdSwitch());
- out.writeLong(sourceNamesystem.getBlockIdManager().getLastAllocatedContiguousBlockId());
+ final BlockIdManager blockIdManager = sourceNamesystem.getBlockManager()
+ .getBlockIdManager();
+ out.writeLong(blockIdManager.getGenerationStampV1());
+ out.writeLong(blockIdManager.getGenerationStampV2());
+ out.writeLong(blockIdManager.getGenerationStampAtblockIdSwitch());
+ out.writeLong(blockIdManager.getLastAllocatedContiguousBlockId());
out.writeLong(context.getTxId());
out.writeLong(sourceNamesystem.dir.getLastInodeId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index ef0cc1d..635dc34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -293,7 +293,7 @@ public final class FSImageFormatProtobuf {
private void loadNameSystemSection(InputStream in) throws IOException {
NameSystemSection s = NameSystemSection.parseDelimitedFrom(in);
- BlockIdManager blockIdManager = fsn.getBlockIdManager();
+ BlockIdManager blockIdManager = fsn.getBlockManager().getBlockIdManager();
blockIdManager.setGenerationStampV1(s.getGenstampV1());
blockIdManager.setGenerationStampV2(s.getGenstampV2());
blockIdManager.setGenerationStampV1Limit(s.getGenstampV1Limit());
@@ -548,7 +548,7 @@ public final class FSImageFormatProtobuf {
throws IOException {
final FSNamesystem fsn = context.getSourceNamesystem();
OutputStream out = sectionOutputStream;
- BlockIdManager blockIdManager = fsn.getBlockIdManager();
+ BlockIdManager blockIdManager = fsn.getBlockManager().getBlockIdManager();
NameSystemSection.Builder b = NameSystemSection.newBuilder()
.setGenstampV1(blockIdManager.getGenerationStampV1())
.setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index fa110e5..3d26544 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -195,7 +195,6 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -236,6 +235,7 @@ import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics;
import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
@@ -316,8 +316,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
NameNodeMXBean {
public static final Log LOG = LogFactory.getLog(FSNamesystem.class);
- private final BlockIdManager blockIdManager;
-
boolean isAuditEnabled() {
return (!isDefaultAuditLogger || auditLog.isInfoEnabled())
&& !auditLoggers.isEmpty();
@@ -557,7 +555,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void clear() {
dir.reset();
dtSecretManager.reset();
- blockIdManager.clear();
leaseManager.removeAllLeases();
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
@@ -570,8 +567,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LeaseManager getLeaseManager() {
return leaseManager;
}
-
- @Override
+
public boolean isHaEnabled() {
return haEnabled;
}
@@ -728,9 +724,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// block manager needs the haEnabled initialized
- this.blockManager = new BlockManager(this, conf);
+ this.blockManager = new BlockManager(this, haEnabled, conf);
this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics();
- this.blockIdManager = new BlockIdManager(blockManager);
// Get the checksum type from config
String checksumTypeStr = conf.get(DFS_CHECKSUM_TYPE_KEY,
@@ -1253,8 +1248,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
getFSImage().editLog.close();
}
}
-
- @Override
+
public void checkOperation(OperationCategory op) throws StandbyException {
if (haContext != null) {
// null in some unit tests
@@ -1542,8 +1536,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public boolean isRunning() {
return fsRunning;
}
-
- @Override
+
public boolean isInStandbyState() {
if (haContext == null || haContext.getState() == null) {
// We're still starting up. In this case, if HA is
@@ -1556,6 +1549,25 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * return a list of blocks & their locations on <code>datanode</code> whose
+ * total size is <code>size</code>
+ *
+ * @param datanode on which blocks are located
+ * @param size total size of blocks
+ */
+ public BlocksWithLocations getBlocks(DatanodeID datanode, long size)
+ throws IOException {
+ checkOperation(OperationCategory.READ);
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ return getBlockManager().getBlocksWithLocations(datanode, size);
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
* Dump all metadata into specified file
*/
void metaSave(String filename) throws IOException {
@@ -3041,7 +3053,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// start recovery of the last block for this file
long blockRecoveryId = nextGenerationStamp(
- blockIdManager.isLegacyBlock(lastBlock));
+ blockManager.isLegacyBlock(lastBlock));
lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
if(copyOnTruncate) {
lastBlock.setGenerationStamp(blockRecoveryId);
@@ -4482,11 +4494,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Increments, logs and then returns the stamp
*/
long nextGenerationStamp(boolean legacyBlock)
- throws IOException, SafeModeException {
+ throws IOException {
assert hasWriteLock();
checkNameNodeSafeMode("Cannot get next generation stamp");
- long gs = blockIdManager.nextGenerationStamp(legacyBlock);
+ long gs = blockManager.nextGenerationStamp(legacyBlock);
if (legacyBlock) {
getEditLog().logGenerationStampV1(gs);
} else {
@@ -4504,8 +4516,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private long nextBlockId(boolean isStriped) throws IOException {
assert hasWriteLock();
checkNameNodeSafeMode("Cannot get next block ID");
- final long blockId = isStriped ?
- blockIdManager.nextStripedBlockId() : blockIdManager.nextContiguousBlockId();
+ final long blockId = blockManager.nextBlockId(isStriped);
getEditLog().logAllocateBlockId(blockId);
// NB: callers sync the log
return blockId;
@@ -4632,7 +4643,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final INodeFile file = checkUCBlock(block, clientName);
// get a new generation stamp and an access token
- block.setGenerationStamp(nextGenerationStamp(blockIdManager.isLegacyBlock(block.getLocalBlock())));
+ block.setGenerationStamp(nextGenerationStamp(
+ blockManager.isLegacyBlock(block.getLocalBlock())));
locatedBlock = BlockManager.newLocatedBlock(
block, file.getLastBlock(), null, -1);
@@ -5472,10 +5484,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return blockManager;
}
- public BlockIdManager getBlockIdManager() {
- return blockIdManager;
- }
-
/** @return the FSDirectory. */
public FSDirectory getFSDirectory() {
return dir;
@@ -5611,11 +5619,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw it;
}
}
-
- @Override
- public boolean isGenStampInFuture(Block block) {
- return blockIdManager.isGenStampInFuture(block);
- }
@VisibleForTesting
public EditLogTailer getEditLogTailer() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 67f1386..60366ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -560,7 +560,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
checkNNStartup();
namesystem.checkSuperuserPrivilege();
- return namesystem.getBlockManager().getBlocks(datanode, size);
+ return namesystem.getBlocks(datanode, size);
}
@Override // NamenodeProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
index f2cc75b..5a9e69b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
@@ -41,14 +41,8 @@ public interface Namesystem extends RwLock, SafeMode {
/** @return the block pool ID */
String getBlockPoolId();
- boolean isInStandbyState();
-
- boolean isGenStampInFuture(Block block);
-
BlockCollection getBlockCollection(long id);
- void checkOperation(OperationCategory read) throws StandbyException;
-
void startSecretManagerIfNecessary();
/**
@@ -68,11 +62,6 @@ public interface Namesystem extends RwLock, SafeMode {
HAContext getHAContext();
/**
- * @return true if the HA is enabled else false
- */
- boolean isHaEnabled();
-
- /**
* @return Whether the namenode is transitioning to active state and is in the
* middle of the starting active services.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 4a74d2f..0e4e167 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -130,7 +130,7 @@ public class TestBlockManager {
Mockito.doReturn(true).when(fsn).hasWriteLock();
Mockito.doReturn(true).when(fsn).hasReadLock();
Mockito.doReturn(true).when(fsn).isRunning();
- bm = new BlockManager(fsn, conf);
+ bm = new BlockManager(fsn, false, conf);
final String[] racks = {
"/rackA",
"/rackA",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index 47d4a43..ade5574 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -94,16 +94,16 @@ public class TestBlockManagerSafeMode {
doReturn(true).when(fsn).hasWriteLock();
doReturn(true).when(fsn).hasReadLock();
doReturn(true).when(fsn).isRunning();
- doReturn(true).when(fsn).isGenStampInFuture(any(Block.class));
NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
- bm = spy(new BlockManager(fsn, conf));
+ bm = spy(new BlockManager(fsn, false, conf));
+ doReturn(true).when(bm).isGenStampInFuture(any(Block.class));
dn = spy(bm.getDatanodeManager());
Whitebox.setInternalState(bm, "datanodeManager", dn);
// the datanode threshold is always met
when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
- bmSafeMode = new BlockManagerSafeMode(bm, fsn, conf);
+ bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 518a359..3259612 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1315,7 +1315,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
FSNamesystem mockNS = mock(FSNamesystem.class);
when(mockNS.hasWriteLock()).thenReturn(true);
when(mockNS.hasReadLock()).thenReturn(true);
- BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
+ BlockManager bm = new BlockManager(mockNS, false, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
BlockInfo block1 = genBlockInfo(ThreadLocalRandom.current().nextLong());
@@ -1365,7 +1365,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
Namesystem mockNS = mock(Namesystem.class);
when(mockNS.hasWriteLock()).thenReturn(true);
- BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
+ BlockManager bm = new BlockManager(mockNS, false, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
long blkID1 = ThreadLocalRandom.current().nextLong();
@@ -1437,7 +1437,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
Namesystem mockNS = mock(Namesystem.class);
when(mockNS.hasReadLock()).thenReturn(true);
- BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
+ BlockManager bm = new BlockManager(mockNS, false, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
BlockInfo block1 = genBlockInfo(ThreadLocalRandom.current().nextLong());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
index 648745c..8d1bbad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
@@ -82,8 +82,8 @@ public class TestSequentialBlockGroupId {
cluster.waitActive();
fs = cluster.getFileSystem();
- blockGrpIdGenerator = cluster.getNamesystem().getBlockIdManager()
- .getBlockGroupIdGenerator();
+ blockGrpIdGenerator = cluster.getNamesystem().getBlockManager()
+ .getBlockIdManager().getBlockGroupIdGenerator();
fs.mkdirs(ecDir);
cluster.getFileSystem().getClient()
.setErasureCodingPolicy("/ecDir", null);
@@ -179,10 +179,10 @@ public class TestSequentialBlockGroupId {
// collision during blockGroup Id generation
FSNamesystem fsn = cluster.getNamesystem();
// Replace SequentialBlockIdGenerator with a spy
- SequentialBlockIdGenerator blockIdGenerator = spy(fsn.getBlockIdManager()
- .getBlockIdGenerator());
- Whitebox.setInternalState(fsn.getBlockIdManager(), "blockIdGenerator",
- blockIdGenerator);
+ SequentialBlockIdGenerator blockIdGenerator = spy(fsn.getBlockManager()
+ .getBlockIdManager().getBlockIdGenerator());
+ Whitebox.setInternalState(fsn.getBlockManager().getBlockIdManager(),
+ "blockIdGenerator", blockIdGenerator);
SequentialBlockIdGenerator spySequentialBlockIdGenerator = new SequentialBlockIdGenerator(
null) {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
index 8235294..e612ea9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockId.java
@@ -116,8 +116,8 @@ public class TestSequentialBlockId {
// Rewind the block ID counter in the name system object. This will result
// in block ID collisions when we try to allocate new blocks.
- SequentialBlockIdGenerator blockIdGenerator = fsn.getBlockIdManager()
- .getBlockIdGenerator();
+ SequentialBlockIdGenerator blockIdGenerator = fsn.getBlockManager()
+ .getBlockIdManager().getBlockIdGenerator();
blockIdGenerator.setCurrentValue(blockIdGenerator.getCurrentValue() - 5);
// Trigger collisions by creating a new file.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 56846af..f2c0330 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -1020,7 +1020,7 @@ public class TestFileTruncate {
assertThat(truncateBlock.getNumBytes(),
is(oldBlock.getNumBytes()));
assertThat(truncateBlock.getGenerationStamp(),
- is(fsn.getBlockIdManager().getGenerationStampV2()));
+ is(fsn.getBlockManager().getBlockIdManager().getGenerationStampV2()));
assertThat(file.getLastBlock().getBlockUCState(),
is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY));
long blockRecoveryId = file.getLastBlock().getUnderConstructionFeature()
@@ -1054,7 +1054,7 @@ public class TestFileTruncate {
assertThat(truncateBlock.getNumBytes() < oldBlock.getNumBytes(),
is(true));
assertThat(truncateBlock.getGenerationStamp(),
- is(fsn.getBlockIdManager().getGenerationStampV2()));
+ is(fsn.getBlockManager().getBlockIdManager().getGenerationStampV2()));
assertThat(file.getLastBlock().getBlockUCState(),
is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY));
long blockRecoveryId = file.getLastBlock().getUnderConstructionFeature()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c304890c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index 6033642..4d37d61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -517,8 +517,8 @@ public class TestSaveNamespace {
FSNamesystem spyFsn = spy(fsn);
final FSNamesystem finalFsn = spyFsn;
DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
- BlockIdManager bid = spy(spyFsn.getBlockIdManager());
- Whitebox.setInternalState(finalFsn, "blockIdManager", bid);
+ BlockIdManager bid = spy(spyFsn.getBlockManager().getBlockIdManager());
+ Whitebox.setInternalState(finalFsn.getBlockManager(), "blockIdManager", bid);
doAnswer(delayer).when(bid).getGenerationStampV2();
ExecutorService pool = Executors.newFixedThreadPool(2);
[47/50] [abbrv] hadoop git commit: HADOOP-12731. Remove useless
boxing/unboxing code. Contributed by Kousuke Saruta.
Posted by ka...@apache.org.
HADOOP-12731. Remove useless boxing/unboxing code. Contributed by Kousuke Saruta.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/736eb17a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/736eb17a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/736eb17a
Branch: refs/heads/YARN-1011
Commit: 736eb17a796a1c1ad5f4db2c6a64f6752db7bec3
Parents: 2b83329
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jan 25 13:47:29 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jan 25 13:47:29 2016 +0900
----------------------------------------------------------------------
.../util/TestZKSignerSecretProvider.java | 2 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/fs/FileContext.java | 4 +-
.../org/apache/hadoop/ha/SshFenceByTcpPort.java | 4 +-
.../apache/hadoop/io/AbstractMapWritable.java | 54 +++++++-------------
.../hadoop/security/ShellBasedIdMapping.java | 9 ++--
.../java/org/apache/hadoop/util/bloom/Key.java | 4 +-
.../apache/hadoop/ha/TestSshFenceByTcpPort.java | 8 +--
.../apache/hadoop/test/GenericTestUtils.java | 2 +-
.../gridmix/DistributedCacheEmulator.java | 4 +-
.../hadoop/mapred/gridmix/CommonJobTest.java | 2 +-
.../mapred/gridmix/TestPseudoLocalFs.java | 2 +-
.../apache/hadoop/streaming/DelayEchoApp.java | 2 +-
13 files changed, 43 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
index 4f8b5ae..8211314 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
@@ -35,7 +35,7 @@ public class TestZKSignerSecretProvider {
// rollover every 2 sec
private final int timeout = 4000;
- private final long rolloverFrequency = Long.valueOf(timeout / 2);
+ private final long rolloverFrequency = timeout / 2;
@Before
public void setup() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 36cac2f..3db68fb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1645,6 +1645,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12730. Hadoop streaming -mapper and -reducer options are wrongly
documented as required. (Kengo Seki via aajisaka)
+ HADOOP-12731. Remove useless boxing/unboxing code.
+ (Kousuke Saruta via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 2456154..d96abad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -761,7 +761,7 @@ public class FileContext {
@Override
public Boolean next(final AbstractFileSystem fs, final Path p)
throws IOException, UnresolvedLinkException {
- return Boolean.valueOf(fs.delete(p, recursive));
+ return fs.delete(p, recursive);
}
}.resolve(this, absF);
}
@@ -895,7 +895,7 @@ public class FileContext {
@Override
public Boolean next(final AbstractFileSystem fs, final Path p)
throws IOException, UnresolvedLinkException {
- return Boolean.valueOf(fs.setReplication(p, replication));
+ return fs.setReplication(p, replication);
}
}.resolve(this, absF);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
index 90eb915..5815564 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
@@ -256,10 +256,10 @@ public class SshFenceByTcpPort extends Configured
}
}
- private Integer parseConfiggedPort(String portStr)
+ private int parseConfiggedPort(String portStr)
throws BadFencingConfigurationException {
try {
- return Integer.valueOf(portStr);
+ return Integer.parseInt(portStr);
} catch (NumberFormatException nfe) {
throw new BadFencingConfigurationException(
"Port number '" + portStr + "' invalid");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
index cc1e517..7dd9e69 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
@@ -116,55 +116,39 @@ public abstract class AbstractMapWritable implements Writable, Configurable {
DataInputBuffer in = new DataInputBuffer();
in.reset(out.getData(), out.getLength());
readFields(in);
-
+
} catch (IOException e) {
throw new IllegalArgumentException("map cannot be copied: " +
e.getMessage());
}
-
+
} else {
throw new IllegalArgumentException("source map cannot be null");
}
}
-
+
/** constructor. */
protected AbstractMapWritable() {
this.conf = new AtomicReference<Configuration>();
- addToMap(ArrayWritable.class,
- Byte.valueOf(Integer.valueOf(-127).byteValue()));
- addToMap(BooleanWritable.class,
- Byte.valueOf(Integer.valueOf(-126).byteValue()));
- addToMap(BytesWritable.class,
- Byte.valueOf(Integer.valueOf(-125).byteValue()));
- addToMap(FloatWritable.class,
- Byte.valueOf(Integer.valueOf(-124).byteValue()));
- addToMap(IntWritable.class,
- Byte.valueOf(Integer.valueOf(-123).byteValue()));
- addToMap(LongWritable.class,
- Byte.valueOf(Integer.valueOf(-122).byteValue()));
- addToMap(MapWritable.class,
- Byte.valueOf(Integer.valueOf(-121).byteValue()));
- addToMap(MD5Hash.class,
- Byte.valueOf(Integer.valueOf(-120).byteValue()));
- addToMap(NullWritable.class,
- Byte.valueOf(Integer.valueOf(-119).byteValue()));
- addToMap(ObjectWritable.class,
- Byte.valueOf(Integer.valueOf(-118).byteValue()));
- addToMap(SortedMapWritable.class,
- Byte.valueOf(Integer.valueOf(-117).byteValue()));
- addToMap(Text.class,
- Byte.valueOf(Integer.valueOf(-116).byteValue()));
- addToMap(TwoDArrayWritable.class,
- Byte.valueOf(Integer.valueOf(-115).byteValue()));
-
- // UTF8 is deprecated so we don't support it
+ addToMap(ArrayWritable.class, (byte)-127);
+ addToMap(BooleanWritable.class, (byte)-126);
+ addToMap(BytesWritable.class, (byte)-125);
+ addToMap(FloatWritable.class, (byte)-124);
+ addToMap(IntWritable.class, (byte)-123);
+ addToMap(LongWritable.class, (byte)-122);
+ addToMap(MapWritable.class, (byte)-121);
+ addToMap(MD5Hash.class, (byte)-120);
+ addToMap(NullWritable.class, (byte)-119);
+ addToMap(ObjectWritable.class, (byte)-118);
+ addToMap(SortedMapWritable.class, (byte)-117);
+ addToMap(Text.class, (byte)-116);
+ addToMap(TwoDArrayWritable.class, (byte)-115);
- addToMap(VIntWritable.class,
- Byte.valueOf(Integer.valueOf(-114).byteValue()));
- addToMap(VLongWritable.class,
- Byte.valueOf(Integer.valueOf(-113).byteValue()));
+ // UTF8 is deprecated so we don't support it
+ addToMap(VIntWritable.class, (byte)-114);
+ addToMap(VLongWritable.class, (byte)-113);
}
/** @return the conf */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index fd362d0..20bccd5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -203,9 +203,8 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
* Integer, e.g. 4294967294 maps to -2 and 4294967295 maps to -1.
*/
private static Integer parseId(final String idStr) {
- Long longVal = Long.parseLong(idStr);
- int intVal = longVal.intValue();
- return Integer.valueOf(intVal);
+ long longVal = Long.parseLong(idStr);
+ return Integer.valueOf((int)longVal);
}
/**
@@ -605,8 +604,8 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
// We know the line is fine to parse without error checking like this
// since it matched the regex above.
String firstComponent = lineMatcher.group(1);
- int remoteId = parseId(lineMatcher.group(2));
- int localId = parseId(lineMatcher.group(3));
+ Integer remoteId = parseId(lineMatcher.group(2));
+ Integer localId = parseId(lineMatcher.group(3));
if (firstComponent.equals("uid")) {
uidMapping.put(localId, remoteId);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
index 7ac134c..e8ad18c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
@@ -177,8 +177,8 @@ public class Key implements WritableComparable<Key> {
}
if (result == 0) {
- result = Double.valueOf(this.weight - other.weight).intValue();
+ result = (int)(this.weight - other.weight);
}
return result;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java
index 4796fe6..fb238fe 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestSshFenceByTcpPort.java
@@ -34,20 +34,20 @@ public class TestSshFenceByTcpPort {
static {
((Log4JLogger)SshFenceByTcpPort.LOG).getLogger().setLevel(Level.ALL);
}
-
+
private static String TEST_FENCING_HOST = System.getProperty(
"test.TestSshFenceByTcpPort.host", "localhost");
private static final String TEST_FENCING_PORT = System.getProperty(
"test.TestSshFenceByTcpPort.port", "8020");
private static final String TEST_KEYFILE = System.getProperty(
"test.TestSshFenceByTcpPort.key");
-
+
private static final InetSocketAddress TEST_ADDR =
new InetSocketAddress(TEST_FENCING_HOST,
- Integer.valueOf(TEST_FENCING_PORT));
+ Integer.parseInt(TEST_FENCING_PORT));
private static final HAServiceTarget TEST_TARGET =
new DummyHAService(HAServiceState.ACTIVE, TEST_ADDR);
-
+
/**
* Connect to Google's DNS server - not running ssh!
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 9c14bb4..7e494a8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -479,6 +479,6 @@ public abstract class GenericTestUtils {
*/
public static void assumeInNativeProfile() {
Assume.assumeTrue(
- Boolean.valueOf(System.getProperty("runningWithNative", "false")));
+ Boolean.parseBoolean(System.getProperty("runningWithNative", "false")));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
index d30a51d..72027c1 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
@@ -330,7 +330,7 @@ class DistributedCacheEmulator {
// Check if visibilities are available because older hadoop versions
// didn't have public, private Distributed Caches separately.
boolean visibility =
- (visibilities == null) ? true : Boolean.valueOf(visibilities[i]);
+ (visibilities == null) || Boolean.parseBoolean(visibilities[i]);
if (isLocalDistCacheFile(files[i], user, visibility)) {
// local FS based distributed cache file.
// Create this file on the pseudo local FS on the fly (i.e. when the
@@ -514,7 +514,7 @@ class DistributedCacheEmulator {
// Check if visibilities are available because older hadoop versions
// didn't have public, private Distributed Caches separately.
boolean visibility =
- (visibilities == null) ? true : Boolean.valueOf(visibilities[i]);
+ (visibilities == null) || Boolean.parseBoolean(visibilities[i]);
if (isLocalDistCacheFile(files[i], user, visibility)) {
// local FS based distributed cache file.
// Create this file on the pseudo local FS.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
index 24e5aad..6a45ba7 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
@@ -172,7 +172,7 @@ public class CommonJobTest {
assertTrue("Gridmix job name is not in the expected format.",
jobName.equals(GridmixJob.JOB_NAME_PREFIX + jobSeqNum));
final FileStatus stat = GridmixTestUtils.dfs.getFileStatus(new Path(
- GridmixTestUtils.DEST, "" + Integer.valueOf(jobSeqNum)));
+ GridmixTestUtils.DEST, "" + Integer.parseInt(jobSeqNum)));
assertEquals("Wrong owner for " + jobName, spec.getUser(),
stat.getOwner());
final int nMaps = spec.getNumberMaps();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
index b9c2728..a607ece 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
@@ -95,7 +95,7 @@ public class TestPseudoLocalFs {
// validate fileSize
String[] parts = path.toUri().getPath().split("\\.");
- long expectedFileSize = Long.valueOf(parts[parts.length - 1]);
+ long expectedFileSize = Long.parseLong(parts[parts.length - 1]);
assertEquals("Invalid file size.", expectedFileSize, stat.getLen());
} else {
assertTrue("getFileStatus() did not throw Exception for invalid file "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736eb17a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/DelayEchoApp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/DelayEchoApp.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/DelayEchoApp.java
index 546b7eb..a912dc9 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/DelayEchoApp.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/DelayEchoApp.java
@@ -45,7 +45,7 @@ public class DelayEchoApp {
int seconds = 5;
if (args.length >= 1) {
try {
- seconds = Integer.valueOf(args[0]);
+ seconds = Integer.parseInt(args[0]);
} catch (NumberFormatException e) {
// just use default 5.
}
[15/50] [abbrv] hadoop git commit: YARN-4603. FairScheduler should
mention user requested queuename in error message when failed in queue ACL
check. (Tao Jie via kasha)
Posted by ka...@apache.org.
YARN-4603. FairScheduler should mention user requested queuename in error message when failed in queue ACL check. (Tao Jie via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4992398a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4992398a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4992398a
Branch: refs/heads/YARN-1011
Commit: 4992398aeeb213ce59e068b0226e22e49ce559eb
Parents: f858ac2
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Thu Jan 21 17:40:59 2016 +0100
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Thu Jan 21 17:40:59 2016 +0100
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/scheduler/fair/FairScheduler.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4992398a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4e36c90..4cbf1a8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -101,6 +101,9 @@ Release 2.9.0 - UNRELEASED
YARN-4559. Make leader elector and zk store share the same curator client.
(Jian He via xgong)
+ YARN-4603. FairScheduler should mention user requested queuename in error
+ message when failed in queue ACL check. (Tao Jie via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4992398a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a36dd8c..7e013e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -656,7 +656,8 @@ public class FairScheduler extends
if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)
&& !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, userUgi)) {
String msg = "User " + userUgi.getUserName() +
- " cannot submit applications to queue " + queue.getName();
+ " cannot submit applications to queue " + queue.getName() +
+ "(requested queuename is " + queueName + ")";
LOG.info(msg);
rmContext.getDispatcher().getEventHandler()
.handle(new RMAppEvent(applicationId,
[16/50] [abbrv] hadoop git commit: YARN-4608. Redundant code
statement in WritingYarnApplications. Contributed by Kai Sasaki.
Posted by ka...@apache.org.
YARN-4608. Redundant code statement in WritingYarnApplications. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ac9511d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ac9511d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ac9511d
Branch: refs/heads/YARN-1011
Commit: 3ac9511da2c0ac815d96713aec099fa87daef648
Parents: 4992398
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jan 22 02:13:02 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Jan 22 02:14:31 2016 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../src/site/markdown/WritingYarnApplications.md | 8 +++-----
2 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ac9511d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4cbf1a8..ca89b32 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1295,6 +1295,9 @@ Release 2.8.0 - UNRELEASED
YARN-4557. Fix improper Queues sorting in PartitionedQueueComparator
when accessible-node-labels=*. (Naganarasimha G R via wangda)
+ YARN-4608. Redundant code statement in WritingYarnApplications.
+ (Kai Sasaki via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ac9511d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
index d5a7a17..1aaec18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
@@ -208,7 +208,7 @@ if (debugFlag) {
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
-// Get final commmand
+// Get final command
StringBuilder command = new StringBuilder();
for (CharSequence str : vargs) {
command.append(str).append(" ");
@@ -339,7 +339,7 @@ ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
nmClientAsync.start();
```
-* The AM has to emit heartbeats to the RM to keep it informed that the AM is alive and still running. The timeout expiry interval at the RM is defined by a config setting accessible via `YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS` with the default being defined by `YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS`. The ApplicationMaster needs to register itself with the ResourceManager to start hearbeating.
+* The AM has to emit heartbeats to the RM to keep it informed that the AM is alive and still running. The timeout expiry interval at the RM is defined by a config setting accessible via `YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS` with the default being defined by `YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS`. The ApplicationMaster needs to register itself with the ResourceManager to start heartbeating.
```java
// Register self with ResourceManager
@@ -386,8 +386,6 @@ LOG.info("Received " + previousAMRunningContainers.size()
```java
List<Container> previousAMRunningContainers =
response.getContainersFromPreviousAttempts();
-List<Container> previousAMRunningContainers =
- response.getContainersFromPreviousAttempts();
LOG.info("Received " + previousAMRunningContainers.size()
+ " previous AM's running containers on AM registration.");
@@ -486,7 +484,7 @@ vargs.add(shellArgs);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
-// Get final commmand
+// Get final command
StringBuilder command = new StringBuilder();
for (CharSequence str : vargs) {
command.append(str).append(" ");
[38/50] [abbrv] hadoop git commit: HDFS-9646. ErasureCodingWorker may
fail when recovering data blocks with length less than the first internal
block. Contributed by Jing Zhao.
Posted by ka...@apache.org.
HDFS-9646. ErasureCodingWorker may fail when recovering data blocks with length less than the first internal block. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95363bcc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95363bcc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95363bcc
Branch: refs/heads/YARN-1011
Commit: 95363bcc7dae28ba9ae2cd7ee9a258fcb58cd932
Parents: 34a3900
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Jan 22 09:46:02 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Jan 22 09:46:02 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSInputStream.java | 48 ++++--
.../hadoop/hdfs/DFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 13 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../erasurecode/ErasureCodingWorker.java | 146 +++++++++++-----
.../server/protocol/BlockECRecoveryCommand.java | 2 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 17 +-
.../hadoop/hdfs/TestRecoverStripedFile.java | 172 ++++++++++++-------
8 files changed, 270 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 3de60b2..3c91ca1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -32,7 +32,6 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
@@ -909,7 +908,8 @@ public class DFSInputStream extends FSInputStream
}
}
- protected synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
+ protected synchronized int readWithStrategy(ReaderStrategy strategy, int off,
+ int len) throws IOException {
dfsClient.checkOpen();
if (closed.get()) {
throw new IOException("Stream closed");
@@ -959,7 +959,7 @@ public class DFSInputStream extends FSInputStream
// Check if need to report block replicas corruption either read
// was successful or ChecksumException occured.
reportCheckSumFailure(corruptedBlockMap,
- currentLocatedBlock.getLocations().length);
+ currentLocatedBlock.getLocations().length, false);
}
}
}
@@ -1492,7 +1492,8 @@ public class DFSInputStream extends FSInputStream
// Check and report if any block replicas are corrupted.
// BlockMissingException may be caught if all block replicas are
// corrupted.
- reportCheckSumFailure(corruptedBlockMap, blk.getLocations().length);
+ reportCheckSumFailure(corruptedBlockMap, blk.getLocations().length,
+ false);
}
remaining -= bytesToRead;
@@ -1508,6 +1509,7 @@ public class DFSInputStream extends FSInputStream
/**
* DFSInputStream reports checksum failure.
+ * For replicated blocks, we have the following logic:
* Case I : client has tried multiple data nodes and at least one of the
* attempts has succeeded. We report the other failures as corrupted block to
* namenode.
@@ -1515,29 +1517,39 @@ public class DFSInputStream extends FSInputStream
* only report if the total number of replica is 1. We do not
* report otherwise since this maybe due to the client is a handicapped client
* (who can not read).
+ *
+ * For erasure-coded blocks, each block in corruptedBlockMap is an internal
+ * block in a block group, and there is usually only one DataNode
+ * corresponding to each internal block. For this case we simply report the
+ * corrupted blocks to NameNode and ignore the above logic.
+ *
* @param corruptedBlockMap map of corrupted blocks
* @param dataNodeCount number of data nodes who contains the block replicas
*/
protected void reportCheckSumFailure(
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap,
- int dataNodeCount) {
+ int dataNodeCount, boolean isStriped) {
if (corruptedBlockMap.isEmpty()) {
return;
}
- Iterator<Entry<ExtendedBlock, Set<DatanodeInfo>>> it = corruptedBlockMap
- .entrySet().iterator();
- Entry<ExtendedBlock, Set<DatanodeInfo>> entry = it.next();
- ExtendedBlock blk = entry.getKey();
- Set<DatanodeInfo> dnSet = entry.getValue();
- if (((dnSet.size() < dataNodeCount) && (dnSet.size() > 0))
- || ((dataNodeCount == 1) && (dnSet.size() == dataNodeCount))) {
- DatanodeInfo[] locs = new DatanodeInfo[dnSet.size()];
- int i = 0;
- for (DatanodeInfo dn:dnSet) {
- locs[i++] = dn;
+ List<LocatedBlock> reportList = new ArrayList<>(corruptedBlockMap.size());
+ for (Map.Entry<ExtendedBlock, Set<DatanodeInfo>> entry :
+ corruptedBlockMap.entrySet()) {
+ ExtendedBlock blk = entry.getKey();
+ Set<DatanodeInfo> dnSet = entry.getValue();
+ if (isStriped || ((dnSet.size() < dataNodeCount) && (dnSet.size() > 0))
+ || ((dataNodeCount == 1) && (dnSet.size() == dataNodeCount))) {
+ DatanodeInfo[] locs = new DatanodeInfo[dnSet.size()];
+ int i = 0;
+ for (DatanodeInfo dn:dnSet) {
+ locs[i++] = dn;
+ }
+ reportList.add(new LocatedBlock(blk, locs));
}
- LocatedBlock [] lblocks = { new LocatedBlock(blk, locs) };
- dfsClient.reportChecksumFailure(src, lblocks);
+ }
+ if (reportList.size() > 0) {
+ dfsClient.reportChecksumFailure(src,
+ reportList.toArray(new LocatedBlock[reportList.size()]));
}
corruptedBlockMap.clear();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 7433256..d15e536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -451,7 +451,7 @@ public class DFSStripedInputStream extends DFSInputStream {
// Check if need to report block replicas corruption either read
// was successful or ChecksumException occured.
reportCheckSumFailure(corruptedBlockMap,
- currentLocatedBlock.getLocations().length);
+ currentLocatedBlock.getLocations().length, true);
}
}
return -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index e8653c8..dbd53a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSStripedOutputStream;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -35,6 +34,8 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.security.token.Token;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.*;
@@ -72,6 +73,8 @@ import java.util.concurrent.TimeUnit;
@InterfaceAudience.Private
public class StripedBlockUtil {
+ public static final Logger LOG = LoggerFactory.getLogger(StripedBlockUtil.class);
+
/**
* This method parses a striped block group into individual blocks.
*
@@ -221,15 +224,11 @@ public class StripedBlockUtil {
return new StripingChunkReadResult(StripingChunkReadResult.TIMEOUT);
}
} catch (ExecutionException e) {
- if (DFSClient.LOG.isDebugEnabled()) {
- DFSClient.LOG.debug("Exception during striped read task", e);
- }
+ LOG.debug("Exception during striped read task", e);
return new StripingChunkReadResult(futures.remove(future),
StripingChunkReadResult.FAILED);
} catch (CancellationException e) {
- if (DFSClient.LOG.isDebugEnabled()) {
- DFSClient.LOG.debug("Exception during striped read task", e);
- }
+ LOG.debug("Exception during striped read task", e);
return new StripingChunkReadResult(futures.remove(future),
StripingChunkReadResult.CANCELLED);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 43adf62..940fa90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -406,6 +406,9 @@ Trunk (Unreleased)
HDFS-9615. Fix variable name typo in DFSConfigKeys. (Ray Chiang via
Arpit Agarwal)
+ HDFS-9646. ErasureCodingWorker may fail when recovering data blocks with
+ length less than the first internal block. (jing9)
+
BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
HDFS-7347. Configurable erasure coding policy for individual files and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 5588eec..6ad7164 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -32,8 +32,10 @@ import java.util.BitSet;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
@@ -46,6 +48,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.BlockReader;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -121,9 +124,8 @@ public final class ErasureCodingWorker {
}
private void initializeStripedReadThreadPool(int num) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using striped reads; pool threads=" + num);
- }
+ LOG.debug("Using striped reads; pool threads=" + num);
+
STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60,
TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new Daemon.DaemonFactory() {
@@ -148,9 +150,7 @@ public final class ErasureCodingWorker {
}
private void initializeStripedBlkRecoveryThreadPool(int num) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using striped block recovery; pool threads=" + num);
- }
+ LOG.debug("Using striped block recovery; pool threads=" + num);
STRIPED_BLK_RECOVERY_THREAD_POOL = new ThreadPoolExecutor(2, num, 60,
TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
new Daemon.DaemonFactory() {
@@ -368,11 +368,11 @@ public final class ErasureCodingWorker {
* @return StripedReader
*/
private StripedReader addStripedReader(int i, long offsetInBlock) {
- StripedReader reader = new StripedReader(liveIndices[i]);
+ final ExtendedBlock block = getBlock(blockGroup, liveIndices[i]);
+ StripedReader reader = new StripedReader(liveIndices[i], block, sources[i]);
stripedReaders.add(reader);
- BlockReader blockReader = newBlockReader(
- getBlock(blockGroup, liveIndices[i]), offsetInBlock, sources[i]);
+ BlockReader blockReader = newBlockReader(block, offsetInBlock, sources[i]);
if (blockReader != null) {
initChecksumAndBufferSizeIfNeeded(blockReader);
reader.blockReader = blockReader;
@@ -435,19 +435,27 @@ public final class ErasureCodingWorker {
throw new IOException(error);
}
- long firstStripedBlockLength = getBlockLen(blockGroup, 0);
- while (positionInBlock < firstStripedBlockLength) {
- int toRead = Math.min(
- bufferSize, (int)(firstStripedBlockLength - positionInBlock));
+ long maxTargetLength = 0;
+ for (short targetIndex : targetIndices) {
+ maxTargetLength = Math.max(maxTargetLength,
+ getBlockLen(blockGroup, targetIndex));
+ }
+ while (positionInBlock < maxTargetLength) {
+ final int toRecover = (int) Math.min(
+ bufferSize, maxTargetLength - positionInBlock);
// step1: read from minimum source DNs required for reconstruction.
- // The returned success list is the source DNs we do real read from
- success = readMinimumStripedData4Recovery(success);
+ // The returned success list is the source DNs we do real read from
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptionMap = new HashMap<>();
+ try {
+ success = readMinimumStripedData4Recovery(success, toRecover,
+ corruptionMap);
+ } finally {
+ // report corrupted blocks to NN
+ reportCorruptedBlocks(corruptionMap);
+ }
// step2: decode to reconstruct targets
- long remaining = firstStripedBlockLength - positionInBlock;
- int toRecoverLen = remaining < bufferSize ?
- (int)remaining : bufferSize;
- recoverTargets(success, targetsStatus, toRecoverLen);
+ recoverTargets(success, targetsStatus, toRecover);
// step3: transfer data
if (transferData2Targets(targetsStatus) == 0) {
@@ -456,7 +464,7 @@ public final class ErasureCodingWorker {
}
clearBuffers();
- positionInBlock += toRead;
+ positionInBlock += toRecover;
}
endTargetBlocks(targetsStatus);
@@ -513,10 +521,11 @@ public final class ErasureCodingWorker {
}
}
- private long getReadLength(int index) {
+ /** the reading length should not exceed the length for recovery */
+ private int getReadLength(int index, int recoverLength) {
long blockLen = getBlockLen(blockGroup, index);
long remaining = blockLen - positionInBlock;
- return remaining > bufferSize ? bufferSize : remaining;
+ return (int) Math.min(remaining, recoverLength);
}
/**
@@ -529,11 +538,15 @@ public final class ErasureCodingWorker {
* operations and next iteration read.
*
* @param success the initial success list of source DNs we think best
+ * @param recoverLength the length to recover.
* @return updated success list of source DNs we do real read
* @throws IOException
*/
- private int[] readMinimumStripedData4Recovery(final int[] success)
+ private int[] readMinimumStripedData4Recovery(final int[] success,
+ int recoverLength, Map<ExtendedBlock, Set<DatanodeInfo>> corruptionMap)
throws IOException {
+ Preconditions.checkArgument(recoverLength >= 0 &&
+ recoverLength <= bufferSize);
int nsuccess = 0;
int[] newSuccess = new int[minRequiredSources];
BitSet used = new BitSet(sources.length);
@@ -543,9 +556,11 @@ public final class ErasureCodingWorker {
*/
for (int i = 0; i < minRequiredSources; i++) {
StripedReader reader = stripedReaders.get(success[i]);
- if (getReadLength(liveIndices[success[i]]) > 0) {
- Callable<Void> readCallable = readFromBlock(
- reader.blockReader, reader.buffer);
+ final int toRead = getReadLength(liveIndices[success[i]],
+ recoverLength);
+ if (toRead > 0) {
+ Callable<Void> readCallable = readFromBlock(reader, reader.buffer,
+ toRead, corruptionMap);
Future<Void> f = readService.submit(readCallable);
futures.put(f, success[i]);
} else {
@@ -570,10 +585,10 @@ public final class ErasureCodingWorker {
StripedReader failedReader = stripedReaders.get(result.index);
closeBlockReader(failedReader.blockReader);
failedReader.blockReader = null;
- resultIndex = scheduleNewRead(used);
+ resultIndex = scheduleNewRead(used, recoverLength, corruptionMap);
} else if (result.state == StripingChunkReadResult.TIMEOUT) {
// If timeout, we also schedule a new read.
- resultIndex = scheduleNewRead(used);
+ resultIndex = scheduleNewRead(used, recoverLength, corruptionMap);
}
if (resultIndex >= 0) {
newSuccess[nsuccess++] = resultIndex;
@@ -601,6 +616,9 @@ public final class ErasureCodingWorker {
}
private void paddingBufferToLen(ByteBuffer buffer, int len) {
+ if (len > buffer.limit()) {
+ buffer.limit(len);
+ }
int toPadding = len - buffer.position();
for (int i = 0; i < toPadding; i++) {
buffer.put((byte) 0);
@@ -648,8 +666,8 @@ public final class ErasureCodingWorker {
int m = 0;
for (int i = 0; i < targetBuffers.length; i++) {
if (targetsStatus[i]) {
+ targetBuffers[i].limit(toRecoverLen);
outputs[m++] = targetBuffers[i];
- outputs[i].limit(toRecoverLen);
}
}
decoder.decode(inputs, erasedIndices, outputs);
@@ -658,7 +676,7 @@ public final class ErasureCodingWorker {
if (targetsStatus[i]) {
long blockLen = getBlockLen(blockGroup, targetIndices[i]);
long remaining = blockLen - positionInBlock;
- if (remaining < 0) {
+ if (remaining <= 0) {
targetBuffers[i].limit(0);
} else if (remaining < toRecoverLen) {
targetBuffers[i].limit((int)remaining);
@@ -678,16 +696,19 @@ public final class ErasureCodingWorker {
* @param used the used source DNs in this iteration.
* @return the array index of source DN if don't need to do real read.
*/
- private int scheduleNewRead(BitSet used) {
+ private int scheduleNewRead(BitSet used, int recoverLength,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptionMap) {
StripedReader reader = null;
// step1: initially we may only have <code>minRequiredSources</code>
// number of StripedReader, and there may be some source DNs we never
// read before, so will try to create StripedReader for one new source DN
// and try to read from it. If found, go to step 3.
int m = stripedReaders.size();
+ int toRead = 0;
while (reader == null && m < sources.length) {
reader = addStripedReader(m, positionInBlock);
- if (getReadLength(liveIndices[m]) > 0) {
+ toRead = getReadLength(liveIndices[m], recoverLength);
+ if (toRead > 0) {
if (reader.blockReader == null) {
reader = null;
m++;
@@ -706,12 +727,14 @@ public final class ErasureCodingWorker {
for (int i = 0; reader == null && i < stripedReaders.size(); i++) {
if (!used.get(i)) {
StripedReader r = stripedReaders.get(i);
- if (getReadLength(liveIndices[i]) > 0) {
+ toRead = getReadLength(liveIndices[i], recoverLength);
+ if (toRead > 0) {
closeBlockReader(r.blockReader);
r.blockReader = newBlockReader(
getBlock(blockGroup, liveIndices[i]), positionInBlock,
sources[i]);
if (r.blockReader != null) {
+ r.buffer.position(0);
m = i;
reader = r;
}
@@ -725,8 +748,8 @@ public final class ErasureCodingWorker {
// step3: schedule if find a correct source DN and need to do real read.
if (reader != null) {
- Callable<Void> readCallable = readFromBlock(
- reader.blockReader, reader.buffer);
+ Callable<Void> readCallable = readFromBlock(reader, reader.buffer,
+ toRead, corruptionMap);
Future<Void> f = readService.submit(readCallable);
futures.put(f, m);
used.set(m);
@@ -742,15 +765,22 @@ public final class ErasureCodingWorker {
}
}
- private Callable<Void> readFromBlock(final BlockReader reader,
- final ByteBuffer buf) {
+ private Callable<Void> readFromBlock(final StripedReader reader,
+ final ByteBuffer buf, final int length,
+ final Map<ExtendedBlock, Set<DatanodeInfo>> corruptionMap) {
return new Callable<Void>() {
@Override
public Void call() throws Exception {
try {
- actualReadFromBlock(reader, buf);
+ buf.limit(length);
+ actualReadFromBlock(reader.blockReader, buf);
return null;
+ } catch (ChecksumException e) {
+ LOG.warn("Found Checksum error for " + reader.block + " from "
+ + reader.source + " at " + e.getPos());
+ addCorruptedBlock(reader.block, reader.source, corruptionMap);
+ throw e;
} catch (IOException e) {
LOG.info(e.getMessage());
throw e;
@@ -760,6 +790,30 @@ public final class ErasureCodingWorker {
};
}
+ private void reportCorruptedBlocks(
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptionMap) throws IOException {
+ if (!corruptionMap.isEmpty()) {
+ for (Map.Entry<ExtendedBlock, Set<DatanodeInfo>> entry :
+ corruptionMap.entrySet()) {
+ for (DatanodeInfo dnInfo : entry.getValue()) {
+ datanode.reportRemoteBadBlock(dnInfo, entry.getKey());
+ }
+ }
+ }
+ }
+
+ private void addCorruptedBlock(ExtendedBlock blk, DatanodeInfo node,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptionMap) {
+ Set<DatanodeInfo> dnSet = corruptionMap.get(blk);
+ if (dnSet == null) {
+ dnSet = new HashSet<>();
+ corruptionMap.put(blk, dnSet);
+ }
+ if (!dnSet.contains(node)) {
+ dnSet.add(node);
+ }
+ }
+
/**
* Read bytes from block
*/
@@ -900,14 +954,14 @@ public final class ErasureCodingWorker {
}
if (zeroStripeBuffers != null) {
- for (int i = 0; i < zeroStripeBuffers.length; i++) {
- zeroStripeBuffers[i].clear();
+ for (ByteBuffer zeroStripeBuffer : zeroStripeBuffers) {
+ zeroStripeBuffer.clear();
}
}
- for (int i = 0; i < targetBuffers.length; i++) {
- if (targetBuffers[i] != null) {
- targetBuffers[i].clear();
+ for (ByteBuffer targetBuffer : targetBuffers) {
+ if (targetBuffer != null) {
+ targetBuffer.clear();
}
}
}
@@ -998,9 +1052,13 @@ public final class ErasureCodingWorker {
private final short index; // internal block index
private BlockReader blockReader;
private ByteBuffer buffer;
+ private final ExtendedBlock block;
+ private final DatanodeInfo source;
- private StripedReader(short index) {
+ StripedReader(short index, ExtendedBlock block, DatanodeInfo source) {
this.index = index;
+ this.block = block;
+ this.source = source;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
index 1cb74b3..d0c1786 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
@@ -136,7 +136,7 @@ public class BlockECRecoveryCommand extends DatanodeCommand {
.append("Recovering ").append(block).append(" From: ")
.append(Arrays.asList(sources)).append(" To: [")
.append(Arrays.asList(targets)).append(")\n")
- .append(" Block Indices: ").append(Arrays.asList(liveBlockIndices))
+ .append(" Block Indices: ").append(Arrays.toString(liveBlockIndices))
.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
index 32b0216..b0af50e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
@@ -29,12 +30,16 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -51,6 +56,14 @@ import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
public class TestReadStripedFileWithDecoding {
static final Log LOG = LogFactory.getLog(TestReadStripedFileWithDecoding.class);
+ static {
+ ((Log4JLogger)LogFactory.getLog(BlockPlacementPolicy.class))
+ .getLogger().setLevel(Level.ALL);
+ GenericTestUtils.setLogLevel(BlockManager.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(BlockManager.blockLog, Level.ALL);
+ GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL);
+ }
+
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
@@ -66,9 +79,9 @@ public class TestReadStripedFileWithDecoding {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, 0);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
- cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
- .numDataNodes(numDNs).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
fs = cluster.getFileSystem();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95363bcc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
index 21352b5..ca9d933 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
@@ -23,11 +23,12 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ThreadLocalRandom;
+import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -64,17 +66,25 @@ public class TestRecoverStripedFile {
static {
GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(BlockManager.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(BlockManager.blockLog, Level.ALL);
+ }
+
+ enum RecoveryType {
+ DataOnly,
+ ParityOnly,
+ Any
}
private MiniDFSCluster cluster;
- private Configuration conf;
private DistributedFileSystem fs;
// Map: DatanodeID -> datanode index in cluster
- private Map<DatanodeID, Integer> dnMap = new HashMap<DatanodeID, Integer>();
+ private Map<DatanodeID, Integer> dnMap = new HashMap<>();
+ private final Random random = new Random();
@Before
public void setup() throws IOException {
- conf = new Configuration();
+ final Configuration conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY,
cellSize - 1);
@@ -104,75 +114,140 @@ public class TestRecoverStripedFile {
@Test(timeout = 120000)
public void testRecoverOneParityBlock() throws Exception {
int fileLen = 10 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverOneParityBlock", fileLen, 0, 1);
+ assertFileBlocksRecovery("/testRecoverOneParityBlock", fileLen,
+ RecoveryType.ParityOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverOneParityBlock1() throws Exception {
int fileLen = cellSize + cellSize/10;
- assertFileBlocksRecovery("/testRecoverOneParityBlock1", fileLen, 0, 1);
+ assertFileBlocksRecovery("/testRecoverOneParityBlock1", fileLen,
+ RecoveryType.ParityOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverOneParityBlock2() throws Exception {
int fileLen = 1;
- assertFileBlocksRecovery("/testRecoverOneParityBlock2", fileLen, 0, 1);
+ assertFileBlocksRecovery("/testRecoverOneParityBlock2", fileLen,
+ RecoveryType.ParityOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverOneParityBlock3() throws Exception {
int fileLen = 3 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverOneParityBlock3", fileLen, 0, 1);
+ assertFileBlocksRecovery("/testRecoverOneParityBlock3", fileLen,
+ RecoveryType.ParityOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverThreeParityBlocks() throws Exception {
int fileLen = 10 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverThreeParityBlocks", fileLen, 0, 3);
+ assertFileBlocksRecovery("/testRecoverThreeParityBlocks", fileLen,
+ RecoveryType.ParityOnly, 3);
}
@Test(timeout = 120000)
public void testRecoverThreeDataBlocks() throws Exception {
int fileLen = 10 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverThreeDataBlocks", fileLen, 1, 3);
+ assertFileBlocksRecovery("/testRecoverThreeDataBlocks", fileLen,
+ RecoveryType.DataOnly, 3);
}
@Test(timeout = 120000)
public void testRecoverThreeDataBlocks1() throws Exception {
int fileLen = 3 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverThreeDataBlocks1", fileLen, 1, 3);
+ assertFileBlocksRecovery("/testRecoverThreeDataBlocks1", fileLen,
+ RecoveryType.DataOnly, 3);
}
@Test(timeout = 120000)
public void testRecoverOneDataBlock() throws Exception {
int fileLen = 10 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverOneDataBlock", fileLen, 1, 1);
+ assertFileBlocksRecovery("/testRecoverOneDataBlock", fileLen,
+ RecoveryType.DataOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverOneDataBlock1() throws Exception {
int fileLen = cellSize + cellSize/10;
- assertFileBlocksRecovery("/testRecoverOneDataBlock1", fileLen, 1, 1);
+ assertFileBlocksRecovery("/testRecoverOneDataBlock1", fileLen,
+ RecoveryType.DataOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverOneDataBlock2() throws Exception {
int fileLen = 1;
- assertFileBlocksRecovery("/testRecoverOneDataBlock2", fileLen, 1, 1);
+ assertFileBlocksRecovery("/testRecoverOneDataBlock2", fileLen,
+ RecoveryType.DataOnly, 1);
}
@Test(timeout = 120000)
public void testRecoverAnyBlocks() throws Exception {
int fileLen = 3 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverAnyBlocks", fileLen, 2, 2);
+ assertFileBlocksRecovery("/testRecoverAnyBlocks", fileLen,
+ RecoveryType.Any, 2);
}
@Test(timeout = 120000)
public void testRecoverAnyBlocks1() throws Exception {
int fileLen = 10 * blockSize + blockSize/10;
- assertFileBlocksRecovery("/testRecoverAnyBlocks1", fileLen, 2, 3);
+ assertFileBlocksRecovery("/testRecoverAnyBlocks1", fileLen,
+ RecoveryType.Any, 3);
}
-
+
+ private int[] generateDeadDnIndices(RecoveryType type, int deadNum,
+ byte[] indices) {
+ List<Integer> deadList = new ArrayList<>(deadNum);
+ while (deadList.size() < deadNum) {
+ int dead = random.nextInt(indices.length);
+ boolean isOfType = true;
+ if (type == RecoveryType.DataOnly) {
+ isOfType = indices[dead] < dataBlkNum;
+ } else if (type == RecoveryType.ParityOnly) {
+ isOfType = indices[dead] >= dataBlkNum;
+ }
+ if (isOfType && !deadList.contains(dead)) {
+ deadList.add(dead);
+ }
+ }
+ int[] d = new int[deadNum];
+ for (int i = 0; i < deadNum; i++) {
+ d[i] = deadList.get(i);
+ }
+ return d;
+ }
+
+ private void shutdownDataNodes(DataNode dn) throws IOException {
+ /*
+ * Kill the datanode which contains one replica
+ * We need to make sure it dead in namenode: clear its update time and
+ * trigger NN to check heartbeat.
+ */
+ dn.shutdown();
+ cluster.setDataNodeDead(dn.getDatanodeId());
+ }
+
+ private int generateErrors(Map<ExtendedBlock, DataNode> corruptTargets,
+ RecoveryType type)
+ throws IOException {
+ int stoppedDN = 0;
+ for (Map.Entry<ExtendedBlock, DataNode> target : corruptTargets.entrySet()) {
+ if (stoppedDN == 0 || type != RecoveryType.DataOnly
+ || random.nextBoolean()) {
+ // stop at least one DN to trigger recovery
+ LOG.info("Note: stop DataNode " + target.getValue().getDisplayName()
+ + " with internal block " + target.getKey());
+ shutdownDataNodes(target.getValue());
+ stoppedDN++;
+ } else { // corrupt the data on the DN
+ LOG.info("Note: corrupt data on " + target.getValue().getDisplayName()
+ + " with internal block " + target.getKey());
+ cluster.corruptReplica(target.getValue(), target.getKey());
+ }
+ }
+ return stoppedDN;
+ }
+
/**
* Test the file blocks recovery.
* 1. Check the replica is recovered in the target datanode,
@@ -180,11 +255,7 @@ public class TestRecoverStripedFile {
* 2. Read the file and verify content.
*/
private void assertFileBlocksRecovery(String fileName, int fileLen,
- int recovery, int toRecoverBlockNum) throws Exception {
- if (recovery != 0 && recovery != 1 && recovery != 2) {
- Assert.fail("Invalid recovery: 0 is to recovery parity blocks,"
- + "1 is to recovery data blocks, 2 is any.");
- }
+ RecoveryType type, int toRecoverBlockNum) throws Exception {
if (toRecoverBlockNum < 1 || toRecoverBlockNum > parityBlkNum) {
Assert.fail("toRecoverBlockNum should be between 1 ~ " + parityBlkNum);
}
@@ -192,7 +263,7 @@ public class TestRecoverStripedFile {
Path file = new Path(fileName);
final byte[] data = new byte[fileLen];
- ThreadLocalRandom.current().nextBytes(data);
+ Arrays.fill(data, (byte) 1);
DFSTestUtil.writeFile(fs, file, data);
StripedFileTestUtil.waitBlockGroupsReported(fs, fileName);
@@ -209,26 +280,10 @@ public class TestRecoverStripedFile {
for (DatanodeInfo storageInfo : storageInfos) {
bitset.set(dnMap.get(storageInfo));
}
-
- int[] toDead = new int[toRecoverBlockNum];
- int n = 0;
- for (int i = 0; i < indices.length; i++) {
- if (n < toRecoverBlockNum) {
- if (recovery == 0) {
- if (indices[i] >= dataBlkNum) {
- toDead[n++] = i;
- }
- } else if (recovery == 1) {
- if (indices[i] < dataBlkNum) {
- toDead[n++] = i;
- }
- } else {
- toDead[n++] = i;
- }
- } else {
- break;
- }
- }
+
+ int[] dead = generateDeadDnIndices(type, toRecoverBlockNum, indices);
+ LOG.info("Note: indices == " + Arrays.toString(indices)
+ + ". Generate errors on datanodes: " + Arrays.toString(dead));
DatanodeInfo[] dataDNs = new DatanodeInfo[toRecoverBlockNum];
int[] deadDnIndices = new int[toRecoverBlockNum];
@@ -236,46 +291,41 @@ public class TestRecoverStripedFile {
File[] replicas = new File[toRecoverBlockNum];
File[] metadatas = new File[toRecoverBlockNum];
byte[][] replicaContents = new byte[toRecoverBlockNum][];
+ Map<ExtendedBlock, DataNode> errorMap = new HashMap<>(dead.length);
for (int i = 0; i < toRecoverBlockNum; i++) {
- dataDNs[i] = storageInfos[toDead[i]];
+ dataDNs[i] = storageInfos[dead[i]];
deadDnIndices[i] = dnMap.get(dataDNs[i]);
-
+
// Check the block replica file on deadDn before it dead.
blocks[i] = StripedBlockUtil.constructInternalBlock(
- lastBlock.getBlock(), cellSize, dataBlkNum, indices[toDead[i]]);
+ lastBlock.getBlock(), cellSize, dataBlkNum, indices[dead[i]]);
+ errorMap.put(blocks[i], cluster.getDataNodes().get(deadDnIndices[i]));
replicas[i] = cluster.getBlockFile(deadDnIndices[i], blocks[i]);
metadatas[i] = cluster.getBlockMetadataFile(deadDnIndices[i], blocks[i]);
// the block replica on the datanode should be the same as expected
assertEquals(replicas[i].length(),
StripedBlockUtil.getInternalBlockLength(
- lastBlock.getBlockSize(), cellSize, dataBlkNum, indices[toDead[i]]));
+ lastBlock.getBlockSize(), cellSize, dataBlkNum, indices[dead[i]]));
assertTrue(metadatas[i].getName().
endsWith(blocks[i].getGenerationStamp() + ".meta"));
+ LOG.info("replica " + i + " locates in file: " + replicas[i]);
replicaContents[i] = DFSTestUtil.readFileAsBytes(replicas[i]);
}
int cellsNum = (fileLen - 1) / cellSize + 1;
int groupSize = Math.min(cellsNum, dataBlkNum) + parityBlkNum;
- for (int i = 0; i < toRecoverBlockNum; i++) {
- /*
- * Kill the datanode which contains one replica
- * We need to make sure it dead in namenode: clear its update time and
- * trigger NN to check heartbeat.
- */
- DataNode dn = cluster.getDataNodes().get(deadDnIndices[i]);
- dn.shutdown();
- cluster.setDataNodeDead(dn.getDatanodeId());
- }
+ // shutdown datanodes or generate corruption
+ int stoppedDN = generateErrors(errorMap, type);
// Check the locatedBlocks of the file again
locatedBlocks = getLocatedBlocks(file);
lastBlock = (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
storageInfos = lastBlock.getLocations();
- assertEquals(storageInfos.length, groupSize - toRecoverBlockNum);
+ assertEquals(storageInfos.length, groupSize - stoppedDN);
int[] targetDNs = new int[dnNum - groupSize];
- n = 0;
+ int n = 0;
for (int i = 0; i < dnNum; i++) {
if (!bitset.get(i)) { // not contain replica of the block.
targetDNs[n++] = i;
@@ -289,9 +339,11 @@ public class TestRecoverStripedFile {
// Check the replica on the new target node.
for (int i = 0; i < toRecoverBlockNum; i++) {
File replicaAfterRecovery = cluster.getBlockFile(targetDNs[i], blocks[i]);
+ LOG.info("replica after recovery " + replicaAfterRecovery);
File metadataAfterRecovery =
cluster.getBlockMetadataFile(targetDNs[i], blocks[i]);
assertEquals(replicaAfterRecovery.length(), replicas[i].length());
+ LOG.info("replica before " + replicas[i]);
assertTrue(metadataAfterRecovery.getName().
endsWith(blocks[i].getGenerationStamp() + ".meta"));
byte[] replicaContentAfterRecovery =
@@ -366,7 +418,7 @@ public class TestRecoverStripedFile {
BlockECRecoveryInfo invalidECInfo = new BlockECRecoveryInfo(
new ExtendedBlock("bp-id", 123456), dataDNs, dnStorageInfo, liveIndices,
ErasureCodingPolicyManager.getSystemDefaultPolicy());
- List<BlockECRecoveryInfo> ecTasks = new ArrayList<BlockECRecoveryInfo>();
+ List<BlockECRecoveryInfo> ecTasks = new ArrayList<>();
ecTasks.add(invalidECInfo);
dataNode.getErasureCodingWorker().processErasureCodingTasks(ecTasks);
}
[09/50] [abbrv] hadoop git commit: YARN-4611. Fix scheduler load
simulator to support multi-layer network location. Contributed by Ming Ma.
Posted by ka...@apache.org.
YARN-4611. Fix scheduler load simulator to support multi-layer network
location. Contributed by Ming Ma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1708a4cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1708a4cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1708a4cd
Branch: refs/heads/YARN-1011
Commit: 1708a4cd2377c56e2cb5738720b7eaf10baf13c8
Parents: 890a2eb
Author: Xuan <xg...@apache.org>
Authored: Wed Jan 20 15:41:50 2016 -0800
Committer: Xuan <xg...@apache.org>
Committed: Wed Jan 20 15:41:50 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java | 11 +++++++++--
.../hadoop/yarn/sls/nodemanager/TestNMSimulator.java | 2 +-
.../org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java | 5 +++++
hadoop-yarn-project/CHANGES.txt | 3 +++
4 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
index 922370c..d950aee 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.tools.rumen.JobTraceReader;
import org.apache.hadoop.tools.rumen.LoggedJob;
import org.apache.hadoop.tools.rumen.LoggedTask;
@@ -43,9 +44,15 @@ import org.codehaus.jackson.map.ObjectMapper;
@Unstable
public class SLSUtils {
+ // hostname includes the network path and the host name. for example
+ // "/default-rack/hostFoo" or "/coreSwitchA/TORSwitchB/hostBar".
+ // the function returns two Strings, the first element is the network
+ // location without "/", the second element is the host name. for example,
+ // {"default-rack", "hostFoo"} or "coreSwitchA/TORSwitchB", "hostBar"
public static String[] getRackHostName(String hostname) {
- hostname = hostname.substring(1);
- return hostname.split("/");
+ NodeBase node = new NodeBase(hostname);
+ return new String[] {node.getNetworkLocation().substring(1),
+ node.getName()};
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
index 2381549..f9a3932 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
@@ -51,7 +51,7 @@ public class TestNMSimulator {
public void testNMSimulator() throws Exception {
// Register one node
NMSimulator node1 = new NMSimulator();
- node1.init("rack1/node1", GB * 10, 10, 0, 1000, rm);
+ node1.init("/rack1/node1", GB * 10, 10, 0, 1000, rm);
node1.middleStep();
int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
index 498615a..f4eda67 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
@@ -29,6 +29,11 @@ public class TestSLSUtils {
String rackHostname[] = SLSUtils.getRackHostName(str);
Assert.assertEquals(rackHostname[0], "rack1");
Assert.assertEquals(rackHostname[1], "node1");
+
+ str = "/rackA/rackB/node1";
+ rackHostname = SLSUtils.getRackHostName(str);
+ Assert.assertEquals(rackHostname[0], "rackA/rackB");
+ Assert.assertEquals(rackHostname[1], "node1");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1708a4cd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ee78d22..212ffa1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -131,6 +131,9 @@ Release 2.9.0 - UNRELEASED
YARN-3446. FairScheduler headroom calculation should exclude nodes in the
blacklist. (Zhihai Xu via kasha)
+ YARN-4611. Fix scheduler load simulator to support multi-layer network
+ location. (Ming Ma via xgong)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
[22/50] [abbrv] hadoop git commit: HADOOP-12651. Replace dev-support
with wrappers to Yetus (aw)
Posted by ka...@apache.org.
HADOOP-12651. Replace dev-support with wrappers to Yetus (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cecad2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cecad2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cecad2d
Branch: refs/heads/YARN-1011
Commit: 8cecad2d564d89d0f56e98de80a6f9b0bcb39d7a
Parents: c304890
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jan 21 12:22:03 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jan 21 12:22:03 2016 -0800
----------------------------------------------------------------------
BUILDING.txt | 2 +-
dev-support/releasedocmaker.py | 580 -----
dev-support/shelldocs.py | 271 ---
dev-support/smart-apply-patch.sh | 187 --
dev-support/test-patch.d/checkstyle.sh | 205 --
dev-support/test-patch.d/shellcheck.sh | 178 --
dev-support/test-patch.d/whitespace.sh | 46 -
dev-support/test-patch.sh | 2814 ----------------------
hadoop-common-project/hadoop-common/pom.xml | 6 +-
9 files changed, 3 insertions(+), 4286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 6a098fa..3eb9e9c 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -39,7 +39,7 @@ and all required tools for testing and building have been installed and configur
Note that from within this docker environment you ONLY have access to the Hadoop source
tree from where you started. So if you need to run
- dev-support/test-patch.sh /path/to/my.patch
+ dev-support/bin/test-patch /path/to/my.patch
then the patch must be placed inside the hadoop source tree.
Known issues:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
deleted file mode 100755
index d2e5dda..0000000
--- a/dev-support/releasedocmaker.py
+++ /dev/null
@@ -1,580 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from glob import glob
-from optparse import OptionParser
-from time import gmtime, strftime
-import pprint
-import os
-import re
-import sys
-import urllib
-import urllib2
-try:
- import json
-except ImportError:
- import simplejson as json
-
-releaseVersion={}
-namePattern = re.compile(r' \([0-9]+\)')
-
-asflicense='''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-def clean(str):
- return tableclean(re.sub(namePattern, "", str))
-
-def formatComponents(str):
- str = re.sub(namePattern, '', str).replace("'", "")
- if str != "":
- ret = str
- else:
- # some markdown parsers don't like empty tables
- ret = "."
- return clean(ret)
-
-# convert to utf-8
-# protect some known md metachars
-# or chars that screw up doxia
-def tableclean(str):
- str=str.encode('utf-8')
- str=str.replace("_","\_")
- str=str.replace("\r","")
- str=str.rstrip()
- return str
-
-# same thing as tableclean,
-# except table metachars are also
-# escaped as well as more
-# things we don't want doxia to
-# screw up
-def notableclean(str):
- str=tableclean(str)
- str=str.replace("|","\|")
- str=str.replace("<","\<")
- str=str.replace(">","\>")
- str=str.replace("*","\*")
- str=str.rstrip()
- return str
-
-# clean output dir
-def cleanOutputDir(dir):
- files = os.listdir(dir)
- for name in files:
- os.remove(os.path.join(dir,name))
- os.rmdir(dir)
-
-def mstr(obj):
- if (obj is None):
- return ""
- return unicode(obj)
-
-def buildindex(title,license):
- versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
- with open("index.md","w") as indexfile:
- if license is True:
- indexfile.write(asflicense)
- for v in versions:
- indexfile.write("* %s v%s\n" % (title,v))
- for k in ("Changes","Release Notes"):
- indexfile.write(" * %s (%s/%s.%s.html)\n" \
- % (k,v,k.upper().replace(" ",""),v))
- indexfile.close()
-
-class GetVersions:
- """ yo """
- def __init__(self,versions, projects):
- versions = versions
- projects = projects
- self.newversions = []
- pp = pprint.PrettyPrinter(indent=4)
- at=0
- end=1
- count=100
- versions.sort()
- print "Looking for %s through %s"%(versions[0],versions[-1])
- for p in projects:
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
- data = json.loads(resp.read())
- for d in data:
- if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
- print "Adding %s to the list" % d['name']
- self.newversions.append(d['name'])
- newlist=list(set(self.newversions))
- self.newversions=newlist
-
- def getlist(self):
- pp = pprint.PrettyPrinter(indent=4)
- return(self.newversions)
-
-class Version:
- """Represents a version number"""
- def __init__(self, data):
- self.mod = False
- self.data = data
- found = re.match('^((\d+)(\.\d+)*).*$', data)
- if (found):
- self.parts = [ int(p) for p in found.group(1).split('.') ]
- else:
- self.parts = []
- # backfill version with zeroes if missing parts
- self.parts.extend((0,) * (3 - len(self.parts)))
-
- def __str__(self):
- if (self.mod):
- return '.'.join([ str(p) for p in self.parts ])
- return self.data
-
- def __cmp__(self, other):
- return cmp(self.parts, other.parts)
-
-class Jira:
- """A single JIRA"""
-
- def __init__(self, data, parent):
- self.key = data['key']
- self.fields = data['fields']
- self.parent = parent
- self.notes = None
- self.incompat = None
- self.reviewed = None
-
- def getId(self):
- return mstr(self.key)
-
- def getDescription(self):
- return mstr(self.fields['description'])
-
- def getReleaseNote(self):
- if (self.notes is None):
- field = self.parent.fieldIdMap['Release Note']
- if (self.fields.has_key(field)):
- self.notes=mstr(self.fields[field])
- else:
- self.notes=self.getDescription()
- return self.notes
-
- def getPriority(self):
- ret = ""
- pri = self.fields['priority']
- if(pri is not None):
- ret = pri['name']
- return mstr(ret)
-
- def getAssignee(self):
- ret = ""
- mid = self.fields['assignee']
- if(mid is not None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getComponents(self):
- if (len(self.fields['components'])>0):
- return ", ".join([ comp['name'] for comp in self.fields['components'] ])
- else:
- return ""
-
- def getSummary(self):
- return self.fields['summary']
-
- def getType(self):
- ret = ""
- mid = self.fields['issuetype']
- if(mid is not None):
- ret = mid['name']
- return mstr(ret)
-
- def getReporter(self):
- ret = ""
- mid = self.fields['reporter']
- if(mid is not None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getProject(self):
- ret = ""
- mid = self.fields['project']
- if(mid is not None):
- ret = mid['key']
- return mstr(ret)
-
- def __cmp__(self,other):
- selfsplit=self.getId().split('-')
- othersplit=other.getId().split('-')
- v1=cmp(selfsplit[0],othersplit[0])
- if (v1!=0):
- return v1
- else:
- if selfsplit[1] < othersplit[1]:
- return True
- elif selfsplit[1] > othersplit[1]:
- return False
- return False
-
- def getIncompatibleChange(self):
- if (self.incompat is None):
- field = self.parent.fieldIdMap['Hadoop Flags']
- self.reviewed=False
- self.incompat=False
- if (self.fields.has_key(field)):
- if self.fields[field]:
- for hf in self.fields[field]:
- if hf['value'] == "Incompatible change":
- self.incompat=True
- if hf['value'] == "Reviewed":
- self.reviewed=True
- return self.incompat
-
- def checkMissingComponent(self):
- if (len(self.fields['components'])>0):
- return False
- return True
-
- def checkMissingAssignee(self):
- if (self.fields['assignee'] is not None):
- return False
- return True
-
- def checkVersionString(self):
- field = self.parent.fieldIdMap['Fix Version/s']
- for h in self.fields[field]:
- found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
- if not found:
- return True
- return False
-
- def getReleaseDate(self,version):
- for j in range(len(self.fields['fixVersions'])):
- if self.fields['fixVersions'][j]==version:
- return(self.fields['fixVersions'][j]['releaseDate'])
- return None
-
-class JiraIter:
- """An Iterator of JIRAs"""
-
- def __init__(self, version, projects):
- self.version = version
- self.projects = projects
- v=str(version).replace("-SNAPSHOT","")
-
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field")
- data = json.loads(resp.read())
-
- self.fieldIdMap = {}
- for part in data:
- self.fieldIdMap[part['name']] = part['id']
-
- self.jiras = []
- at=0
- end=1
- count=100
- while (at < end):
- params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
- data = json.loads(resp.read())
- if (data.has_key('errorMessages')):
- raise Exception(data['errorMessages'])
- at = data['startAt'] + data['maxResults']
- end = data['total']
- self.jiras.extend(data['issues'])
-
- needaversion=False
- if v not in releaseVersion:
- needaversion=True
-
- if needaversion is True:
- for i in range(len(data['issues'])):
- for j in range(len(data['issues'][i]['fields']['fixVersions'])):
- if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
- releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
- data['issues'][i]['fields']['fixVersions'][j]['releaseDate']
-
- self.iter = self.jiras.__iter__()
-
- def __iter__(self):
- return self
-
- def next(self):
- data = self.iter.next()
- j = Jira(data, self)
- return j
-
-class Outputs:
- """Several different files to output to at the same time"""
-
- def __init__(self, base_file_name, file_name_pattern, keys, params={}):
- self.params = params
- self.base = open(base_file_name%params, 'w')
- self.others = {}
- for key in keys:
- both = dict(params)
- both['key'] = key
- self.others[key] = open(file_name_pattern%both, 'w')
-
- def writeAll(self, pattern):
- both = dict(self.params)
- both['key'] = ''
- self.base.write(pattern%both)
- for key in self.others.keys():
- both = dict(self.params)
- both['key'] = key
- self.others[key].write(pattern%both)
-
- def writeKeyRaw(self, key, str):
- self.base.write(str)
- if (self.others.has_key(key)):
- self.others[key].write(str)
-
- def close(self):
- self.base.close()
- for fd in self.others.values():
- fd.close()
-
- def writeList(self, mylist):
- for jira in sorted(mylist):
- line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' \
- % (notableclean(jira.getId()), notableclean(jira.getId()),
- notableclean(jira.getSummary()),
- notableclean(jira.getPriority()),
- formatComponents(jira.getComponents()),
- notableclean(jira.getReporter()),
- notableclean(jira.getAssignee()))
- self.writeKeyRaw(jira.getProject(), line)
-
-def main():
- parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
- epilog=
- "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
- " named after the highest version provided.")
- parser.add_option("-i","--index", dest="index", action="store_true",
- default=False, help="build an index file")
- parser.add_option("-l","--license", dest="license", action="store_false",
- default=True, help="Add an ASF license")
- parser.add_option("-n","--lint", dest="lint", action="store_true",
- help="use lint flag to exit on failures")
- parser.add_option("-p", "--project", dest="projects",
- action="append", type="string",
- help="projects in JIRA to include in releasenotes", metavar="PROJECT")
- parser.add_option("-r", "--range", dest="range", action="store_true",
- default=False, help="Given versions are a range")
- parser.add_option("-t", "--projecttitle", dest="title",
- type="string",
- help="Title to use for the project (default is Apache PROJECT)")
- parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
- default=False, help="use current date for unreleased versions")
- parser.add_option("-v", "--version", dest="versions",
- action="append", type="string",
- help="versions in JIRA to include in releasenotes", metavar="VERSION")
- (options, args) = parser.parse_args()
-
- if (options.versions is None):
- options.versions = []
-
- if (len(args) > 2):
- options.versions.append(args[2])
-
- if (len(options.versions) <= 0):
- parser.error("At least one version needs to be supplied")
-
- proxy = urllib2.ProxyHandler()
- opener = urllib2.build_opener(proxy)
- urllib2.install_opener(opener)
-
- projects = options.projects
-
- if (options.range is True):
- versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
- else:
- versions = [ Version(v) for v in options.versions ]
- versions.sort();
-
- if (options.title is None):
- title=projects[0]
- else:
- title=options.title
-
- haderrors=False
-
- for v in versions:
- vstr=str(v)
- jlist = JiraIter(vstr,projects)
-
- if vstr in releaseVersion:
- reldate=releaseVersion[vstr]
- elif options.usetoday:
- reldate=strftime("%Y-%m-%d", gmtime())
- else:
- reldate="Unreleased"
-
- if not os.path.exists(vstr):
- os.mkdir(vstr)
-
- reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
- "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
- [], {"ver":v, "date":reldate, "title":title})
- choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
- "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
- [], {"ver":v, "date":reldate, "title":title})
-
- if (options.license is True):
- reloutputs.writeAll(asflicense)
- choutputs.writeAll(asflicense)
-
- relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
- 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
- chhead = '# %(title)s Changelog\n\n' \
- '## Release %(ver)s - %(date)s\n'\
- '\n'
-
- reloutputs.writeAll(relhead)
- choutputs.writeAll(chhead)
- errorCount=0
- warningCount=0
- lintMessage=""
- incompatlist=[]
- buglist=[]
- improvementlist=[]
- newfeaturelist=[]
- subtasklist=[]
- tasklist=[]
- testlist=[]
- otherlist=[]
-
- for jira in sorted(jlist):
- if jira.getIncompatibleChange():
- incompatlist.append(jira)
- elif jira.getType() == "Bug":
- buglist.append(jira)
- elif jira.getType() == "Improvement":
- improvementlist.append(jira)
- elif jira.getType() == "New Feature":
- newfeaturelist.append(jira)
- elif jira.getType() == "Sub-task":
- subtasklist.append(jira)
- elif jira.getType() == "Task":
- tasklist.append(jira)
- elif jira.getType() == "Test":
- testlist.append(jira)
- else:
- otherlist.append(jira)
-
- line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
- % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
- notableclean(jira.getSummary()))
-
- if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
- warningCount+=1
- reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
- reloutputs.writeKeyRaw(jira.getProject(), line)
- line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
- lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
- reloutputs.writeKeyRaw(jira.getProject(), line)
-
- if jira.checkVersionString():
- warningCount+=1
- lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
-
- if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
- errorCount+=1
- errorMessage=[]
- jira.checkMissingComponent() and errorMessage.append("component")
- jira.checkMissingAssignee() and errorMessage.append("assignee")
- lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
-
- if (len(jira.getReleaseNote())>0):
- reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
- reloutputs.writeKeyRaw(jira.getProject(), line)
- line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
- reloutputs.writeKeyRaw(jira.getProject(), line)
-
- if (options.lint is True):
- print lintMessage
- print "======================================="
- print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
- if (errorCount>0):
- haderrors=True
- cleanOutputDir(vstr)
- continue
-
- reloutputs.writeAll("\n\n")
- reloutputs.close()
-
- choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(incompatlist)
-
- choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(newfeaturelist)
-
- choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(improvementlist)
-
- choutputs.writeAll("\n\n### BUG FIXES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(buglist)
-
- choutputs.writeAll("\n\n### TESTS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(testlist)
-
- choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(subtasklist)
-
- choutputs.writeAll("\n\n### OTHER:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(otherlist)
- choutputs.writeList(tasklist)
-
- choutputs.writeAll("\n\n")
- choutputs.close()
-
- if options.index:
- buildindex(title,options.license)
-
- if haderrors is True:
- sys.exit(1)
-
-if __name__ == "__main__":
- main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/shelldocs.py
----------------------------------------------------------------------
diff --git a/dev-support/shelldocs.py b/dev-support/shelldocs.py
deleted file mode 100755
index fc7601a..0000000
--- a/dev-support/shelldocs.py
+++ /dev/null
@@ -1,271 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import sys
-import string
-from optparse import OptionParser
-
-asflicense='''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-def docstrip(key,string):
- string=re.sub("^## @%s " % key ,"",string)
- string=string.lstrip()
- string=string.rstrip()
- return string
-
-def toc(list):
- tocout=[]
- header=()
- for i in list:
- if header != i.getinter():
- header=i.getinter()
- line=" * %s\n" % (i.headerbuild())
- tocout.append(line)
- line=" * [%s](#%s)\n" % (i.getname().replace("_","\_"),i.getname())
- tocout.append(line)
- return tocout
-
-class ShellFunction:
- def __init__(self):
- self.reset()
-
- def __cmp__(self,other):
- if (self.audience == other.audience):
- if (self.stability == other.stability):
- if (self.replaceb == other.replaceb):
- return(cmp(self.name,other.name))
- else:
- if (self.replaceb == "Yes"):
- return -1
- else:
- return 1
- else:
- if (self.stability == "Stable"):
- return -1
- else:
- return 1
- else:
- if (self.audience == "Public"):
- return -1
- else:
- return 1
-
- def reset(self):
- self.name=None
- self.audience=None
- self.stability=None
- self.replaceb=None
- self.returnt=None
- self.desc=None
- self.params=None
-
- def setname(self,text):
- definition=text.split();
- self.name=definition[1]
-
- def getname(self):
- if (self.name is None):
- return "None"
- else:
- return self.name
-
- def setaudience(self,text):
- self.audience=docstrip("audience",text)
- self.audience=self.audience.capitalize()
-
- def getaudience(self):
- if (self.audience is None):
- return "None"
- else:
- return self.audience
-
- def setstability(self,text):
- self.stability=docstrip("stability",text)
- self.stability=self.stability.capitalize()
-
- def getstability(self):
- if (self.stability is None):
- return "None"
- else:
- return self.stability
-
- def setreplace(self,text):
- self.replaceb=docstrip("replaceable",text)
- self.replaceb=self.replaceb.capitalize()
-
- def getreplace(self):
- if (self.replaceb is None):
- return "None"
- else:
- return self.replaceb
-
- def getinter(self):
- return( (self.getaudience(), self.getstability(), self.getreplace()))
-
- def addreturn(self,text):
- if (self.returnt is None):
- self.returnt = []
- self.returnt.append(docstrip("return",text))
-
- def getreturn(self):
- if (self.returnt is None):
- return "Nothing"
- else:
- return "\n\n".join(self.returnt)
-
- def adddesc(self,text):
- if (self.desc is None):
- self.desc = []
- self.desc.append(docstrip("description",text))
-
- def getdesc(self):
- if (self.desc is None):
- return "None"
- else:
- return " ".join(self.desc)
-
- def addparam(self,text):
- if (self.params is None):
- self.params = []
- self.params.append(docstrip("param",text))
-
- def getparams(self):
- if (self.params is None):
- return ""
- else:
- return " ".join(self.params)
-
- def getusage(self):
- line="%s %s" % (self.name, self.getparams())
- return line
-
- def headerbuild(self):
- if self.getreplace() == "Yes":
- replacetext="Replaceable"
- else:
- replacetext="Not Replaceable"
- line="%s/%s/%s" % (self.getaudience(), self.getstability(), replacetext)
- return(line)
-
- def getdocpage(self):
- line="### `%s`\n\n"\
- "* Synopsis\n\n"\
- "```\n%s\n"\
- "```\n\n" \
- "* Description\n\n" \
- "%s\n\n" \
- "* Returns\n\n" \
- "%s\n\n" \
- "| Classification | Level |\n" \
- "| :--- | :--- |\n" \
- "| Audience | %s |\n" \
- "| Stability | %s |\n" \
- "| Replaceable | %s |\n\n" \
- % (self.getname(),
- self.getusage(),
- self.getdesc(),
- self.getreturn(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
-
- def __str__(self):
- line="{%s %s %s %s}" \
- % (self.getname(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
-
-def main():
- parser=OptionParser(usage="usage: %prog --skipprnorep --output OUTFILE --input INFILE [--input INFILE ...]")
- parser.add_option("-o","--output", dest="outfile",
- action="store", type="string",
- help="file to create", metavar="OUTFILE")
- parser.add_option("-i","--input", dest="infile",
- action="append", type="string",
- help="file to read", metavar="INFILE")
- parser.add_option("--skipprnorep", dest="skipprnorep",
- action="store_true", help="Skip Private & Not Replaceable")
-
- (options, args)=parser.parse_args()
-
- allfuncs=[]
- for filename in options.infile:
- with open(filename,"r") as shellcode:
- funcdef=ShellFunction()
- for line in shellcode:
- if line.startswith('## @description'):
- funcdef.adddesc(line)
- elif line.startswith('## @audience'):
- funcdef.setaudience(line)
- elif line.startswith('## @stability'):
- funcdef.setstability(line)
- elif line.startswith('## @replaceable'):
- funcdef.setreplace(line)
- elif line.startswith('## @param'):
- funcdef.addparam(line)
- elif line.startswith('## @return'):
- funcdef.addreturn(line)
- elif line.startswith('function'):
- funcdef.setname(line)
- if options.skipprnorep and \
- funcdef.getaudience() == "Private" and \
- funcdef.getreplace() == "No":
- pass
- else:
- allfuncs.append(funcdef)
- funcdef=ShellFunction()
-
- allfuncs=sorted(allfuncs)
-
- outfile=open(options.outfile, "w")
- outfile.write(asflicense)
- for line in toc(allfuncs):
- outfile.write(line)
-
- outfile.write("\n------\n\n")
-
- header=[]
- for funcs in allfuncs:
- if header != funcs.getinter():
- header=funcs.getinter()
- line="## %s\n" % (funcs.headerbuild())
- outfile.write(line)
- outfile.write(funcs.getdocpage())
- outfile.close()
-
-if __name__ == "__main__":
- main()
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
deleted file mode 100755
index ebcb660..0000000
--- a/dev-support/smart-apply-patch.sh
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Determine if the git diff patch file has prefixes.
-# These files are generated via "git diff" *without* the --no-prefix option.
-#
-# We can apply these patches more easily because we know that the a/ and b/
-# prefixes in the "diff" lines stands for the project root directory.
-# So we don't have to hunt for the project root.
-# And of course, we know that the patch file was generated using git, so we
-# know git apply can handle it properly.
-#
-# Arguments: git diff file name.
-# Return: 0 if it is a git diff with prefix; 1 otherwise.
-#
-has_prefix() {
- awk '/^diff --git / { if ($3 !~ "^a/" || $4 !~ "^b/") { exit 1 } }
- /^\+{3}|-{3} / { if ($2 !~ "^[ab]/" && $2 !~ "^/dev/null") { exit 1 } }' "$1"
- return $?
-}
-
-PATCH_FILE=$1
-DRY_RUN=$2
-if [ -z "$PATCH_FILE" ]; then
- echo usage: $0 patch-file
- exit 1
-fi
-
-TMPDIR=${TMPDIR:-/tmp}
-PATCH=${PATCH:-patch} # allow overriding patch binary
-
-# Cleanup handler for temporary files
-TOCLEAN=""
-cleanup() {
- rm $TOCLEAN
- exit $1
-}
-trap "cleanup 1" HUP INT QUIT TERM
-
-# Allow passing "-" for stdin patches
-if [ "$PATCH_FILE" == "-" ]; then
- PATCH_FILE="$TMPDIR/smart-apply.in.$RANDOM"
- cat /dev/fd/0 > $PATCH_FILE
- TOCLEAN="$TOCLEAN $PATCH_FILE"
-fi
-
-ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
-if [[ ${PATCH_FILE} =~ ^http || ${PATCH_FILE} =~ ${ISSUE_RE} ]]; then
- # Allow downloading of patches
- PFILE="$TMPDIR/smart-apply.in.$RANDOM"
- TOCLEAN="$TOCLEAN $PFILE"
- if [[ ${PATCH_FILE} =~ ^http ]]; then
- patchURL="${PATCH_FILE}"
- else # Get URL of patch from JIRA
- wget -q -O "${PFILE}" "http://issues.apache.org/jira/browse/${PATCH_FILE}"
- if [[ $? != 0 ]]; then
- echo "Unable to determine what ${PATCH_FILE} may reference." 1>&2
- cleanup 1
- elif [[ $(grep -c 'Patch Available' "${PFILE}") == 0 ]]; then
- echo "${PATCH_FILE} is not \"Patch Available\". Exiting." 1>&2
- cleanup 1
- fi
- relativePatchURL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PFILE}" | grep -v -e 'htm[l]*$' | sort | tail -1 | grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
- patchURL="http://issues.apache.org${relativePatchURL}"
- fi
- if [[ -n $DRY_RUN ]]; then
- echo "Downloading ${patchURL}"
- fi
- wget -q -O "${PFILE}" "${patchURL}"
- if [[ $? != 0 ]]; then
- echo "${PATCH_FILE} could not be downloaded." 1>&2
- cleanup 1
- fi
- PATCH_FILE="${PFILE}"
-fi
-
-# Case for git-diff patches
-if grep -q "^diff --git" "${PATCH_FILE}"; then
- GIT_FLAGS="--binary -v"
- if has_prefix "$PATCH_FILE"; then
- GIT_FLAGS="$GIT_FLAGS -p1"
- else
- GIT_FLAGS="$GIT_FLAGS -p0"
- fi
- if [[ -z $DRY_RUN ]]; then
- GIT_FLAGS="$GIT_FLAGS --stat --apply"
- echo Going to apply git patch with: git apply "${GIT_FLAGS}"
- else
- GIT_FLAGS="$GIT_FLAGS --check"
- fi
- # shellcheck disable=SC2086
- git apply ${GIT_FLAGS} "${PATCH_FILE}"
- if [[ $? == 0 ]]; then
- cleanup 0
- fi
- echo "git apply failed. Going to apply the patch with: ${PATCH}"
-fi
-
-# Come up with a list of changed files into $TMP
-TMP="$TMPDIR/smart-apply.paths.$RANDOM"
-TOCLEAN="$TOCLEAN $TMP"
-
-if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
- PLEVEL=0
- #if the patch applied at P0 there is the possability that all we are doing
- # is adding new files and they would apply anywhere. So try to guess the
- # correct place to put those files.
-
- TMP2="$TMPDIR/smart-apply.paths.2.$RANDOM"
- TOCLEAN="$TOCLEAN $TMP2"
-
- egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort -u > $TMP2
-
- if [ ! -s $TMP2 ]; then
- echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
- cleanup 1
- fi
-
- #first off check that all of the files do not exist
- FOUND_ANY=0
- for CHECK_FILE in $(cat $TMP2)
- do
- if [[ -f $CHECK_FILE ]]; then
- FOUND_ANY=1
- fi
- done
-
- if [[ "$FOUND_ANY" = "0" ]]; then
- #all of the files are new files so we have to guess where the correct place to put it is.
-
- # if all of the lines start with a/ or b/, then this is a git patch that
- # was generated without --no-prefix
- if ! grep -qv '^a/\|^b/' $TMP2 ; then
- echo Looks like this is a git patch. Stripping a/ and b/ prefixes
- echo and incrementing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
- sed -i -e 's,^[ab]/,,' $TMP2
- fi
-
- PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 $TMP2 | sort -u)
-
- # if we are at the project root then nothing more to do
- if [[ -d hadoop-common-project ]]; then
- echo Looks like this is being run at project root
-
- # if all of the lines start with hadoop-common/, hadoop-hdfs/, hadoop-yarn/ or hadoop-mapreduce/, this is
- # relative to the hadoop root instead of the subproject root, so we need
- # to chop off another layer
- elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-yarn-project|hadoop-mapreduce-project)$ ]]; then
-
- echo Looks like this is relative to project root. Increasing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
-
- elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-yarn-project\|hadoop-mapreduce-project' ; then
- echo Looks like this is a cross-subproject patch. Try applying from the project root
- cleanup 1
- fi
- fi
-elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=1
-elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=2
-else
- echo "The patch does not appear to apply with p0 to p2";
- cleanup 1;
-fi
-
-# If this is a dry run then exit instead of applying the patch
-if [[ -n $DRY_RUN ]]; then
- cleanup 0;
-fi
-
-echo Going to apply patch with: $PATCH -p$PLEVEL
-$PATCH -p$PLEVEL -E < $PATCH_FILE
-
-cleanup $?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.d/checkstyle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
deleted file mode 100755
index 6311584..0000000
--- a/dev-support/test-patch.d/checkstyle.sh
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin checkstyle
-
-CHECKSTYLE_TIMER=0
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function checkstyle_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test checkstyle
- fi
-}
-
-function checkstyle_mvnrunner
-{
- local logfile=$1
- local output=$2
- local tmp=${PATCH_DIR}/$$.${RANDOM}
- local j
-
- "${MVN}" clean test checkstyle:checkstyle -DskipTests \
- -Dcheckstyle.consoleOutput=true \
- "-D${PROJECT_NAME}PatchProcess" 2>&1 \
- | tee "${logfile}" \
- | ${GREP} ^/ \
- | ${SED} -e "s,${BASEDIR},.,g" \
- > "${tmp}"
-
- # the checkstyle output files are massive, so
- # let's reduce the work by filtering out files
- # that weren't changed. Some modules are
- # MASSIVE and this can cut the output down to
- # by orders of magnitude!!
- for j in ${CHANGED_FILES}; do
- ${GREP} "${j}" "${tmp}" >> "${output}"
- done
-
- rm "${tmp}" 2>/dev/null
-}
-
-function checkstyle_preapply
-{
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
-
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: prepatch"
-
- start_clock
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
- add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
- fi
- popd >/dev/null
- done
-
- # keep track of how much as elapsed for us already
- CHECKSTYLE_TIMER=$(stop_clock)
- return 0
-}
-
-function checkstyle_calcdiffs
-{
- local orig=$1
- local new=$2
- local diffout=$3
- local tmp=${PATCH_DIR}/cs.$$.${RANDOM}
- local count=0
- local j
-
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
-
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1 >> "${diffout}"
- done
-
- if [[ -f "${diffout}" ]]; then
- # shellcheck disable=SC2016
- count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
- fi
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
- echo "${count}"
-}
-
-function checkstyle_postapply
-{
- local rc=0
- local module
- local modules=${CHANGED_MODULES}
- local module_suffix
- local numprepatch=0
- local numpostpatch=0
- local diffpostpatch=0
-
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: postpatch"
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${CHECKSTYLE_TIMER}"
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch checkstyle compilation is broken."
- add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
- continue
- fi
-
- #shellcheck disable=SC2016
- diffpostpatch=$(checkstyle_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
- "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
-
- if [[ ${diffpostpatch} -gt 0 ]] ; then
- ((rc = rc + 1))
-
- # shellcheck disable=SC2016
- numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
- # shellcheck disable=SC2016
- numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
-
- add_jira_table -1 checkstyle "The applied patch generated "\
- "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
- footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
- fi
-
- popd >/dev/null
- done
-
- if [[ ${rc} -gt 0 ]] ; then
- add_jira_footer checkstyle "${footer}"
- return 1
- fi
- add_jira_table +1 checkstyle "There were no new checkstyle issues."
- return 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
deleted file mode 100755
index 9277ea5..0000000
--- a/dev-support/test-patch.d/shellcheck.sh
+++ /dev/null
@@ -1,178 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin shellcheck
-
-SHELLCHECK_TIMER=0
-
-SHELLCHECK=${SHELLCHECK:-$(which shellcheck 2>/dev/null)}
-
-SHELLCHECK_SPECIFICFILES=""
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function shellcheck_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.sh$ ]]; then
- add_test shellcheck
- SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
- fi
-
- if [[ ! ${filename} =~ \. ]]; then
- add_test shellcheck
- fi
-}
-
-function shellcheck_private_findbash
-{
- local i
- local value
- local list
-
- while read line; do
- value=$(find "${line}" ! -name '*.cmd' -type f \
- | ${GREP} -E -v '(.orig$|.rej$)')
- list="${list} ${value}"
- done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
- # shellcheck disable=SC2086
- echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
-}
-
-function shellcheck_preapply
-{
- local i
-
- verify_needed_test shellcheck
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "shellcheck plugin: prepatch"
-
- if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- return 0
- fi
-
- start_clock
-
- # shellcheck disable=SC2016
- SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
-
- echo "Running shellcheck against all identifiable shell scripts"
- pushd "${BASEDIR}" >/dev/null
- for i in $(shellcheck_private_findbash); do
- if [[ -f ${i} ]]; then
- ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
- fi
- done
- popd > /dev/null
- # keep track of how much as elapsed for us already
- SHELLCHECK_TIMER=$(stop_clock)
- return 0
-}
-
-function shellcheck_calcdiffs
-{
- local orig=$1
- local new=$2
- local diffout=$3
- local tmp=${PATCH_DIR}/sc.$$.${RANDOM}
- local count=0
- local j
-
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
-
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1 >> "${diffout}"
- done
-
- if [[ -f "${diffout}" ]]; then
- # shellcheck disable=SC2016
- count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
- fi
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
- echo "${count}"
-}
-
-function shellcheck_postapply
-{
- local i
-
- verify_needed_test shellcheck
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "shellcheck plugin: postpatch"
-
- if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- add_jira_table 0 shellcheck "Shellcheck was not available."
- return 0
- fi
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${SHELLCHECK_TIMER}"
-
- echo "Running shellcheck against all identifiable shell scripts"
- # we re-check this in case one has been added
- for i in $(shellcheck_private_findbash); do
- ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
- done
-
- # shellcheck disable=SC2016
- numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
- # shellcheck disable=SC2016
- numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
-
- diffPostpatch=$(shellcheck_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
- "${PATCH_DIR}/patchshellcheck-result.txt" \
- "${PATCH_DIR}/diffpatchshellcheck.txt"
- )
-
- if [[ ${diffPostpatch} -gt 0 ]] ; then
- add_jira_table -1 shellcheck "The applied patch generated "\
- "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
- add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
- return 1
- fi
-
- add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
- return 0
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.d/whitespace.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
deleted file mode 100755
index 324481c..0000000
--- a/dev-support/test-patch.d/whitespace.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin whitespace
-
-function whitespace_postapply
-{
- local count
- local j
-
- big_console_header "Checking for whitespace at the end of lines"
- start_clock
-
- pushd "${BASEDIR}" >/dev/null
- for j in ${CHANGED_FILES}; do
- ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
- done
-
- # shellcheck disable=SC2016
- count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
-
- if [[ ${count} -gt 0 ]]; then
- add_jira_table -1 whitespace "The patch has ${count}"\
- " line(s) that end in whitespace. Use git apply --whitespace=fix."
- add_jira_footer whitespace "@@BASE@@/whitespace.txt"
- popd >/dev/null
- return 1
- fi
-
- popd >/dev/null
- add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
- return 0
-}
[37/50] [abbrv] hadoop git commit: YARN-4574. Fix random failure in
TestAMRMClientOnRMRestart. (Takashi Ohnishi via rohithsharmaks)
Posted by ka...@apache.org.
YARN-4574. Fix random failure in TestAMRMClientOnRMRestart. (Takashi Ohnishi via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34a39007
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34a39007
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34a39007
Branch: refs/heads/YARN-1011
Commit: 34a3900773df6d9075dcf5476e8fe5208cd8f806
Parents: 2fd19b9
Author: rohithsharmaks <ro...@apache.org>
Authored: Fri Jan 22 21:17:26 2016 +0530
Committer: rohithsharmaks <ro...@apache.org>
Committed: Fri Jan 22 21:17:37 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34a39007/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b667b5b..a4f9874 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -146,6 +146,9 @@ Release 2.9.0 - UNRELEASED
YARN-4497. RM might fail to restart when recovering apps whose attempts are missing.
(Jun Gong via rohithsharmaks)
+ YARN-4574. Fix random failure in TestAMRMClientOnRMRestart.
+ (Takashi Ohnishi via rohithsharmaks)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34a39007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index 0460f1e..0890396 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -219,6 +219,7 @@ public class TestAMRMClientOnRMRestart {
// request
nm1.nodeHeartbeat(containerId.getApplicationAttemptId(),
containerId.getContainerId(), ContainerState.RUNNING);
+ dispatcher.await();
amClient.requestContainerResourceChange(
container, Resource.newInstance(2048, 1));
it.remove();
[04/50] [abbrv] hadoop git commit: HDFS-6054. MiniQJMHACluster should
not use static port to avoid binding failure in unit test. (Yongjun Zhang)
Posted by ka...@apache.org.
HDFS-6054. MiniQJMHACluster should not use static port to avoid binding failure in unit test. (Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57d0a943
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57d0a943
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57d0a943
Branch: refs/heads/YARN-1011
Commit: 57d0a94305b22a469062c7ce85b3159fee17a28b
Parents: 1acc509
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Tue Jan 19 22:54:47 2016 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Tue Jan 19 22:54:47 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 15 +++++--
.../namenode/ha/TestFailureToReadEdits.java | 41 ++++++++++++++++----
3 files changed, 47 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 327cf73..b051f2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2599,6 +2599,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9623. Update example configuration of block state change log in
log4j.properties. (Masatake Iwasaki via aajisaka)
+ HDFS-6054. MiniQJMHACluster should not use static port to avoid binding
+ failure in unit test. (Yongjun Zhang)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
index 8e838c3..0764f12 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
@@ -42,7 +42,6 @@ public class MiniQJMHACluster {
public static final String NAMESERVICE = "ns1";
private static final Random RANDOM = new Random();
- private int basePort = 10000;
public static class Builder {
private final Configuration conf;
@@ -92,9 +91,12 @@ public class MiniQJMHACluster {
private MiniQJMHACluster(Builder builder) throws IOException {
this.conf = builder.conf;
int retryCount = 0;
+ int basePort = 10000;
+
while (true) {
try {
basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ LOG.info("Set MiniQJMHACluster basePort to " + basePort);
// start 3 journal nodes
journalCluster = new MiniJournalCluster.Builder(conf).format(true)
.build();
@@ -104,7 +106,7 @@ public class MiniQJMHACluster {
// start cluster with specified NameNodes
MiniDFSNNTopology topology = createDefaultTopology(builder.numNNs, basePort);
- initHAConf(journalURI, builder.conf, builder.numNNs);
+ initHAConf(journalURI, builder.conf, builder.numNNs, basePort);
// First start up the NNs just to format the namespace. The MinIDFSCluster
// has no way to just format the NameNodes without also starting them.
@@ -123,16 +125,21 @@ public class MiniQJMHACluster {
// restart the cluster
cluster.restartNameNodes();
- ++retryCount;
break;
} catch (BindException e) {
+ if (cluster != null) {
+ cluster.shutdown(true);
+ cluster = null;
+ }
+ ++retryCount;
LOG.info("MiniQJMHACluster port conflicts, retried " +
retryCount + " times");
}
}
}
- private Configuration initHAConf(URI journalURI, Configuration conf, int numNNs) {
+ private Configuration initHAConf(URI journalURI, Configuration conf,
+ int numNNs, int basePort) {
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
journalURI.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57d0a943/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index f82b616..5221ef9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -27,10 +27,14 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
import java.io.IOException;
+import java.net.BindException;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
+import java.util.Random;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -62,11 +66,14 @@ import com.google.common.collect.ImmutableList;
@RunWith(Parameterized.class)
public class TestFailureToReadEdits {
+ private static final Log LOG =
+ LogFactory.getLog(TestFailureToReadEdits.class);
private static final String TEST_DIR1 = "/test1";
private static final String TEST_DIR2 = "/test2";
private static final String TEST_DIR3 = "/test3";
-
+ private static final Random RANDOM = new Random();
+
private final TestType clusterType;
private Configuration conf;
private MiniDFSCluster cluster;
@@ -103,14 +110,32 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
HAUtil.setAllowStandbyReads(conf, true);
-
+
if (clusterType == TestType.SHARED_DIR_HA) {
- MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000);
- cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .checkExitOnShutdown(false)
- .build();
+ int basePort = 10000;
+ int retryCount = 0;
+ while (true) {
+ try {
+ basePort = 10000 + RANDOM.nextInt(1000) * 4;
+ LOG.info("Set SHARED_DIR_HA cluster's basePort to " + basePort);
+ MiniDFSNNTopology topology =
+ MiniQJMHACluster.createDefaultTopology(basePort);
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology)
+ .numDataNodes(0)
+ .checkExitOnShutdown(false)
+ .build();
+ break;
+ } catch (BindException e) {
+ if (cluster != null) {
+ cluster.shutdown(true);
+ cluster = null;
+ }
+ ++retryCount;
+ LOG.info("SHARED_DIR_HA: MiniQJMHACluster port conflicts, retried " +
+ retryCount + " times " + e);
+ }
+ }
} else {
Builder builder = new MiniQJMHACluster.Builder(conf);
builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false);
[19/50] [abbrv] hadoop git commit: YARN-4610. Reservations continue
looking for one app causes other apps to starve. Contributed by Jason Lowe
Posted by ka...@apache.org.
YARN-4610. Reservations continue looking for one app causes other apps to starve. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/468a53b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/468a53b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/468a53b2
Branch: refs/heads/YARN-1011
Commit: 468a53b22f4ac5bb079dff986ba849a687d709fe
Parents: 1bb31fb
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Jan 21 18:30:42 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Jan 21 18:31:29 2016 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/capacity/LeafQueue.java | 6 +-
.../scheduler/capacity/TestReservations.java | 147 ++++++++++++++++++-
3 files changed, 152 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/468a53b2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ca89b32..f789bcb 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1368,6 +1368,9 @@ Release 2.7.3 - UNRELEASED
YARN-4581. AHS writer thread leak makes RM crash while RM is recovering.
(sandflee via junping_du)
+ YARN-4610. Reservations continue looking for one app causes other apps to
+ starve (jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/468a53b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 9c6d8ee..9e64b42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1152,9 +1152,11 @@ public class LeafQueue extends AbstractCSQueue {
@Private
protected synchronized boolean canAssignToUser(Resource clusterResource,
String userName, Resource limit, FiCaSchedulerApp application,
- String nodePartition, ResourceLimits currentResoureLimits) {
+ String nodePartition, ResourceLimits currentResourceLimits) {
User user = getUser(userName);
+ currentResourceLimits.setAmountNeededUnreserve(Resources.none());
+
// Note: We aren't considering the current request since there is a fixed
// overhead of the AM, but it's a > check, not a >= check, so...
if (Resources
@@ -1181,7 +1183,7 @@ public class LeafQueue extends AbstractCSQueue {
Resources.subtract(user.getUsed(nodePartition), limit);
// we can only acquire a new container if we unreserve first to
// respect user-limit
- currentResoureLimits.setAmountNeededUnreserve(amountNeededToUnreserve);
+ currentResourceLimits.setAmountNeededUnreserve(amountNeededToUnreserve);
return true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/468a53b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 49de478..9b920d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -100,12 +100,17 @@ public class TestReservations {
}
private void setup(CapacitySchedulerConfiguration csConf) throws Exception {
+ setup(csConf, false);
+ }
+
+ private void setup(CapacitySchedulerConfiguration csConf,
+ boolean addUserLimits) throws Exception {
csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
final String newRoot = "root" + System.currentTimeMillis();
// final String newRoot = "root";
- setupQueueConfiguration(csConf, newRoot);
+ setupQueueConfiguration(csConf, newRoot, addUserLimits);
YarnConfiguration conf = new YarnConfiguration();
cs.setConf(conf);
@@ -146,7 +151,7 @@ public class TestReservations {
private static final String A = "a";
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf,
- final String newRoot) {
+ final String newRoot, boolean addUserLimits) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT,
@@ -167,6 +172,10 @@ public class TestReservations {
conf.setMaximumCapacity(Q_A, 100);
conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
+ if (addUserLimits) {
+ conf.setUserLimit(Q_A, 25);
+ conf.setUserLimitFactor(Q_A, 0.25f);
+ }
}
static LeafQueue stubLeafQueue(LeafQueue queue) {
@@ -334,6 +343,140 @@ public class TestReservations {
assertEquals(0, app_0.getTotalRequiredResources(priorityReduce));
}
+ // Test that hitting a reservation limit and needing to unreserve
+ // does not affect assigning containers for other users
+ @Test
+ public void testReservationLimitOtherUsers() throws Exception {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ setup(csConf, true);
+
+ // Manipulate queue 'a'
+ LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
+
+ // Users
+ final String user_0 = "user_0";
+ final String user_1 = "user_1";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId_0 = TestUtils
+ .getMockApplicationAttemptId(0, 0);
+ FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
+ mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
+ rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
+
+ a.submitApplicationAttempt(app_0, user_0);
+
+ final ApplicationAttemptId appAttemptId_1 = TestUtils
+ .getMockApplicationAttemptId(1, 0);
+ FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_1, a,
+ mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
+ rmContext.getRMApps().put(app_1.getApplicationId(), mock(RMApp.class));
+
+ a.submitApplicationAttempt(app_1, user_1);
+
+ // Setup some nodes
+ String host_0 = "host_0";
+ FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+ 8 * GB);
+ String host_1 = "host_1";
+ FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
+ 8 * GB);
+ String host_2 = "host_2";
+ FiCaSchedulerNode node_2 = TestUtils.getMockNode(host_2, DEFAULT_RACK, 0,
+ 8 * GB);
+
+ when(csContext.getNode(node_0.getNodeID())).thenReturn(node_0);
+ when(csContext.getNode(node_1.getNodeID())).thenReturn(node_1);
+ when(csContext.getNode(node_2.getNodeID())).thenReturn(node_2);
+
+ cs.getAllNodes().put(node_0.getNodeID(), node_0);
+ cs.getAllNodes().put(node_1.getNodeID(), node_1);
+ cs.getAllNodes().put(node_2.getNodeID(), node_2);
+
+ final int numNodes = 3;
+ Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+
+ // Setup resource-requests
+ Priority priorityAM = TestUtils.createMockPriority(1);
+ Priority priorityMap = TestUtils.createMockPriority(5);
+ Priority priorityReduce = TestUtils.createMockPriority(10);
+
+ app_0.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true,
+ priorityAM, recordFactory)));
+ app_1.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true,
+ priorityAM, recordFactory)));
+
+ // Start testing...
+ // Only AM
+ a.assignContainers(clusterResource, node_0,
+ new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ assertEquals(2 * GB, a.getUsedResources().getMemory());
+ assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(0 * GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0 * GB, a.getMetrics().getReservedMB());
+ assertEquals(2 * GB, a.getMetrics().getAllocatedMB());
+ assertEquals(22 * GB, a.getMetrics().getAvailableMB());
+ assertEquals(2 * GB, node_0.getUsedResource().getMemory());
+ assertEquals(0 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+
+ a.assignContainers(clusterResource, node_1,
+ new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ assertEquals(4 * GB, a.getUsedResources().getMemory());
+ assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(2 * GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0 * GB, a.getMetrics().getReservedMB());
+ assertEquals(4 * GB, a.getMetrics().getAllocatedMB());
+ assertEquals(20 * GB, a.getMetrics().getAvailableMB());
+ assertEquals(2 * GB, node_0.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+
+ // Add a few requests to each app
+ app_0.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 8 * GB, 2, true,
+ priorityMap, recordFactory)));
+ app_1.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 2 * GB, 2, true,
+ priorityMap, recordFactory)));
+
+ // add a reservation for app_0
+ a.assignContainers(clusterResource, node_0,
+ new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ assertEquals(12 * GB, a.getUsedResources().getMemory());
+ assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(2 * GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(8 * GB, a.getMetrics().getReservedMB());
+ assertEquals(4 * GB, a.getMetrics().getAllocatedMB());
+ assertEquals(12 * GB, a.getMetrics().getAvailableMB());
+ assertEquals(2 * GB, node_0.getUsedResource().getMemory());
+ assertEquals(2 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+
+ // next assignment is beyond user limit for user_0 but it should assign to
+ // app_1 for user_1
+ a.assignContainers(clusterResource, node_1,
+ new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ assertEquals(14 * GB, a.getUsedResources().getMemory());
+ assertEquals(2 * GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(4 * GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(8 * GB, a.getMetrics().getReservedMB());
+ assertEquals(6 * GB, a.getMetrics().getAllocatedMB());
+ assertEquals(10 * GB, a.getMetrics().getAvailableMB());
+ assertEquals(2 * GB, node_0.getUsedResource().getMemory());
+ assertEquals(4 * GB, node_1.getUsedResource().getMemory());
+ assertEquals(0 * GB, node_2.getUsedResource().getMemory());
+ }
+
@Test
public void testReservationNoContinueLook() throws Exception {
// Test that with reservations-continue-look-all-nodes feature off
[32/50] [abbrv] hadoop git commit: HDFS-9682. Fix a typo "aplication"
in HttpFS document. Contributed by Weiwei yang.
Posted by ka...@apache.org.
HDFS-9682. Fix a typo "aplication" in HttpFS document. Contributed by Weiwei yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68316346
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68316346
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68316346
Branch: refs/heads/YARN-1011
Commit: 683163468219107afb57a002f2cf2a369442171d
Parents: 8f58f74
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jan 22 20:12:18 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Jan 22 20:12:18 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68316346/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
index ac8f4ca..3e8daa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
@@ -23,7 +23,7 @@ HttpFS can be used to access data in HDFS on a cluster behind of a firewall (the
HttpFS can be used to access data in HDFS using HTTP utilities (such as curl and wget) and HTTP libraries Perl from other languages than Java.
-The **webhdfs** client FileSytem implementation can be used to access HttpFS using the Hadoop filesystem command (`hadoop fs`) line tool as well as from Java aplications using the Hadoop FileSystem Java API.
+The **webhdfs** client FileSytem implementation can be used to access HttpFS using the Hadoop filesystem command (`hadoop fs`) line tool as well as from Java applications using the Hadoop FileSystem Java API.
HttpFS has built-in security supporting Hadoop pseudo authentication and HTTP SPNEGO Kerberos and other pluggable authentication mechanims. It also provides Hadoop proxy user support.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68316346/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8c26ee7..c7061e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2619,6 +2619,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6054. MiniQJMHACluster should not use static port to avoid binding
failure in unit test. (Yongjun Zhang)
+ HDFS-9682. Fix a typo "aplication" in HttpFS document.
+ (Weiwei Yang via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
[02/50] [abbrv] hadoop git commit: HADOOP-12605. Fix intermittent
failure of TestIPC.testIpcWithReaderQueuing (iwasakims)
Posted by ka...@apache.org.
HADOOP-12605. Fix intermittent failure of TestIPC.testIpcWithReaderQueuing (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9eec6cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9eec6cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9eec6cbe
Branch: refs/heads/YARN-1011
Commit: 9eec6cbedcc53e6dd306a4a578a22937ae239260
Parents: 5ff5f67
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Jan 20 05:00:02 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Jan 20 05:00:02 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/ipc/TestIPC.java | 18 ++++++++++++------
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9eec6cbe/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a5dcf6a..85f304a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -676,6 +676,9 @@ Release 2.9.0 - UNRELEASED
BUG FIXES
+ HADOOP-12605. Fix intermittent failure of TestIPC.testIpcWithReaderQueuing
+ (iwasakims)
+
HADOOP-12655. TestHttpServer.testBindAddress bind port range is wider
than expected. (Wei-Chiu Chuang via stevel)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9eec6cbe/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index d6c0658..78dcdcd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -27,6 +27,8 @@ import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
@@ -71,6 +73,7 @@ import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.RPC.RpcKind;
+import org.apache.hadoop.ipc.Server.Call;
import org.apache.hadoop.ipc.Server.Connection;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
import org.apache.hadoop.net.ConnectTimeoutException;
@@ -84,6 +87,7 @@ import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -703,6 +707,7 @@ public class TestIPC {
// goal is to jam a handler with a connection, fill the callq with
// connections, in turn jamming the readers - then flood the server and
// ensure that the listener blocks when the reader connection queues fill
+ @SuppressWarnings("unchecked")
private void checkBlocking(int readers, int readerQ, int callQ) throws Exception {
int handlers = 1; // makes it easier
@@ -722,6 +727,9 @@ public class TestIPC {
// start server
final TestServerQueue server =
new TestServerQueue(clients, readers, callQ, handlers, conf);
+ CallQueueManager<Call> spy = spy(
+ (CallQueueManager<Call>)Whitebox.getInternalState(server, "callQueue"));
+ Whitebox.setInternalState(server, "callQueue", spy);
final InetSocketAddress addr = NetUtils.getConnectAddress(server);
server.start();
@@ -757,12 +765,10 @@ public class TestIPC {
if (i==0) {
// let first reader block in a call
server.firstCallLatch.await();
- } else if (i <= callQ) {
- // let subsequent readers jam the callq, will happen immediately
- while (server.getCallQueueLen() != i) {
- Thread.sleep(1);
- }
- } // additional threads block the readers trying to add to the callq
+ }
+ // wait until reader put a call to callQueue, to make sure all readers
+ // are blocking on the queue after initialClients threads are started.
+ verify(spy, timeout(100).times(i + 1)).put(Mockito.<Call>anyObject());
}
try {
[44/50] [abbrv] hadoop git commit: YARN-4613. Fix test failure in
TestClientRMService#testGetClusterNodes. (Takashi Ohnishi via rohithsharmaks)
Posted by ka...@apache.org.
YARN-4613. Fix test failure in TestClientRMService#testGetClusterNodes. (Takashi Ohnishi via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10dc2c04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10dc2c04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10dc2c04
Branch: refs/heads/YARN-1011
Commit: 10dc2c049304671f0ed9fd737a30cd843427b53e
Parents: 99829eb
Author: rohithsharmaks <ro...@apache.org>
Authored: Sun Jan 24 23:36:15 2016 +0530
Committer: rohithsharmaks <ro...@apache.org>
Committed: Sun Jan 24 23:36:15 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/TestClientRMService.java | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10dc2c04/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1e9f83c..6674194 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -152,6 +152,9 @@ Release 2.9.0 - UNRELEASED
YARN-4574. Fix random failure in TestAMRMClientOnRMRestart.
(Takashi Ohnishi via rohithsharmaks)
+ YARN-4613. Fix test failure in TestClientRMService#testGetClusterNodes.
+ (Takashi Ohnishi via rohithsharmaks)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10dc2c04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 0f892cd..df95957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -209,6 +209,7 @@ public class TestClientRMService {
// Now make the node unhealthy.
node.nodeHeartbeat(false);
+ rm.NMwaitForState(node.getNodeId(), NodeState.UNHEALTHY);
// Call again
nodeReports = client.getClusterNodes(request).getNodeReports();
[13/50] [abbrv] hadoop git commit: HDFS-9634. webhdfs client side
exceptions don't provide enough details. Contributed by Eric Payne.
Posted by ka...@apache.org.
HDFS-9634. webhdfs client side exceptions don't provide enough details. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3616c7b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3616c7b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3616c7b8
Branch: refs/heads/YARN-1011
Commit: 3616c7b855962014750a3259a64c6e2a147da884
Parents: 446987e
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Jan 20 13:18:25 2016 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Jan 21 09:32:57 2016 -0600
----------------------------------------------------------------------
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 17 ++++++++++++++++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 14 +++++++-------
3 files changed, 26 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3616c7b8/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 1fa95c2..f9c2c6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -24,6 +24,7 @@ import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
@@ -502,6 +503,7 @@ public class WebHdfsFileSystem extends FileSystem
new ExcludeDatanodesParam("");
private boolean checkRetry;
+ private String redirectHost;
protected AbstractRunner(final HttpOpParam.Op op, boolean redirected) {
this.op = op;
@@ -553,7 +555,7 @@ public class WebHdfsFileSystem extends FileSystem
*/
protected HttpURLConnection connect(URL url) throws IOException {
//redirect hostname and port
- String redirectHost = null;
+ redirectHost = null;
// resolve redirects for a DN operation unless already resolved
@@ -658,6 +660,19 @@ public class WebHdfsFileSystem extends FileSystem
throw it;
}
} catch (IOException ioe) {
+ // Attempt to include the redirected node in the exception. If the
+ // attempt to recreate the exception fails, just use the original.
+ String node = redirectHost;
+ if (node == null) {
+ node = url.getAuthority();
+ }
+ try {
+ ioe = ioe.getClass().getConstructor(String.class)
+ .newInstance(node + ": " + ioe.getMessage());
+ } catch (NoSuchMethodException | SecurityException
+ | InstantiationException | IllegalAccessException
+ | IllegalArgumentException | InvocationTargetException e) {
+ }
shouldRetry(ioe, retry);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3616c7b8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f1e6bdc..f41537e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2624,6 +2624,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9569. Log the name of the fsimage being loaded for better
supportability. (Yongjun Zhang)
+ HDFS-9634. webhdfs client side exceptions don't provide enough details
+ (Eric Payne via kihwal)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3616c7b8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
index bd4d693..3a87d42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTimeouts.java
@@ -115,7 +115,7 @@ public class TestWebHdfsTimeouts {
fs.listFiles(new Path("/"), false);
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("connect timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": connect timed out", e.getMessage());
}
}
@@ -128,7 +128,7 @@ public class TestWebHdfsTimeouts {
fs.listFiles(new Path("/"), false);
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("Read timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": Read timed out", e.getMessage());
}
}
@@ -143,7 +143,7 @@ public class TestWebHdfsTimeouts {
fs.getDelegationToken("renewer");
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("connect timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": connect timed out", e.getMessage());
}
}
@@ -157,7 +157,7 @@ public class TestWebHdfsTimeouts {
fs.getDelegationToken("renewer");
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("Read timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": Read timed out", e.getMessage());
}
}
@@ -172,7 +172,7 @@ public class TestWebHdfsTimeouts {
fs.getFileChecksum(new Path("/file"));
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("connect timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": connect timed out", e.getMessage());
}
}
@@ -187,7 +187,7 @@ public class TestWebHdfsTimeouts {
fs.getFileChecksum(new Path("/file"));
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("Read timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": Read timed out", e.getMessage());
}
}
@@ -203,7 +203,7 @@ public class TestWebHdfsTimeouts {
os = fs.create(new Path("/file"));
fail("expected timeout");
} catch (SocketTimeoutException e) {
- assertEquals("connect timed out", e.getMessage());
+ assertEquals(fs.getUri().getAuthority() + ": connect timed out", e.getMessage());
} finally {
IOUtils.cleanup(LOG, os);
}
[11/50] [abbrv] hadoop git commit: HDFS-9625. set replication for
empty file failed when set storage policy (Contributed by DENG FEI)
Posted by ka...@apache.org.
HDFS-9625. set replication for empty file failed when set storage policy (Contributed by DENG FEI)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7372b71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7372b71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7372b71
Branch: refs/heads/YARN-1011
Commit: b7372b7166a13111b98794602ca4c166dfd78d29
Parents: 9eec6cb
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 21 19:17:05 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 21 19:17:05 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++++
.../apache/hadoop/hdfs/TestSetrepIncreasing.java | 19 +++++++++++++++++++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7372b71/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e6022fe..f1e6bdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2655,6 +2655,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
createRbw (ade via vinayakumarb)
+ HDFS-9625. set replication for empty file failed when set storage policy
+ (DENG FEI via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7372b71/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 6ca7a03..a00591f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -811,6 +811,10 @@ public class FSDirectory implements Closeable {
long dsDelta, short oldRep, short newRep) {
EnumCounters<StorageType> typeSpaceDeltas =
new EnumCounters<StorageType>(StorageType.class);
+ // empty file
+ if(dsDelta == 0){
+ return typeSpaceDeltas;
+ }
// Storage type and its quota are only available when storage policy is set
if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
BlockStoragePolicy storagePolicy = getBlockManager().getStoragePolicy(storagePolicyID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7372b71/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
index 0af1988..a055adc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java
@@ -83,4 +83,23 @@ public class TestSetrepIncreasing {
public void testSetrepIncreasingSimulatedStorage() throws IOException {
setrep(3, 7, true);
}
+
+ @Test
+ public void testSetRepWithStoragePolicyOnEmptyFile() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ try {
+ Path d = new Path("/tmp");
+ dfs.mkdirs(d);
+ dfs.setStoragePolicy(d, "HOT");
+ Path f = new Path(d, "foo");
+ dfs.createNewFile(f);
+ dfs.setReplication(f, (short) 4);
+ } finally {
+ dfs.close();
+ cluster.shutdown();
+ }
+ }
}
[27/50] [abbrv] hadoop git commit: YARN-4584. RM startup failure when
AM attempts greater than max-attempts. (Bibin A Chundatt via rohithsharmaks)
Posted by ka...@apache.org.
YARN-4584. RM startup failure when AM attempts greater than max-attempts. (Bibin A Chundatt via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3066810
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3066810
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3066810
Branch: refs/heads/YARN-1011
Commit: e30668106dc246f68db36fbd1f2db6ec08cd96f2
Parents: b2ffcc2
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Fri Jan 22 10:14:46 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Fri Jan 22 10:14:46 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 23 +++++---
.../server/resourcemanager/TestRMRestart.java | 58 ++++++++++++++++++++
3 files changed, 77 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3066810/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f789bcb..a7a63b1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -137,6 +137,9 @@ Release 2.9.0 - UNRELEASED
YARN-4611. Fix scheduler load simulator to support multi-layer network
location. (Ming Ma via xgong)
+ YARN-4584. RM startup failure when AM attempts greater than max-attempts.
+ (Bibin A Chundatt via rohithsharmaks)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3066810/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 6ecc7d3..1a390df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -841,7 +841,7 @@ public class RMAppImpl implements RMApp, Recoverable {
this.startTime = appState.getStartTime();
this.callerContext = appState.getCallerContext();
// If interval > 0, some attempts might have been deleted.
- if (submissionContext.getAttemptFailuresValidityInterval() > 0) {
+ if (this.attemptFailuresValidityInterval > 0) {
this.firstAttemptIdInStateStore = appState.getFirstAttemptId();
this.nextAttemptId = firstAttemptIdInStateStore;
}
@@ -1341,7 +1341,9 @@ public class RMAppImpl implements RMApp, Recoverable {
+ "is " + numberOfFailure + ". The max attempts is "
+ app.maxAppAttempts);
- removeExcessAttempts(app);
+ if (app.attemptFailuresValidityInterval > 0) {
+ removeExcessAttempts(app);
+ }
if (!app.submissionContext.getUnmanagedAM()
&& numberOfFailure < app.maxAppAttempts) {
@@ -1381,15 +1383,22 @@ public class RMAppImpl implements RMApp, Recoverable {
}
private void removeExcessAttempts(RMAppImpl app) {
- while (app.nextAttemptId - app.firstAttemptIdInStateStore
- > app.maxAppAttempts) {
+ while (app.nextAttemptId
+ - app.firstAttemptIdInStateStore > app.maxAppAttempts) {
// attempts' first element is oldest attempt because it is a
// LinkedHashMap
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
app.getApplicationId(), app.firstAttemptIdInStateStore);
- app.firstAttemptIdInStateStore++;
- LOG.info("Remove attempt from state store : " + attemptId);
- app.rmContext.getStateStore().removeApplicationAttempt(attemptId);
+ RMAppAttempt rmAppAttempt = app.getRMAppAttempt(attemptId);
+ long endTime = app.systemClock.getTime();
+ if (rmAppAttempt.getFinishTime() < (endTime
+ - app.attemptFailuresValidityInterval)) {
+ app.firstAttemptIdInStateStore++;
+ LOG.info("Remove attempt from state store : " + attemptId);
+ app.rmContext.getStateStore().removeApplicationAttempt(attemptId);
+ } else {
+ break;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3066810/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index bad68f4..625c3b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -42,6 +42,8 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -104,6 +106,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -121,6 +125,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
public class TestRMRestart extends ParameterizedSchedulerTestBase {
+ private static final Log LOG = LogFactory.getLog(TestRMRestart.class);
private final static File TEMP_DIR = new File(System.getProperty(
"test.build.data", "/tmp"), "decommision");
private File hostFile = new File(TEMP_DIR + File.separator + "hostFile.txt");
@@ -2321,4 +2326,57 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
rm2.stop();
}
+ @Test(timeout = 120000)
+ public void testRMRestartAfterPreemption() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+ if (!getSchedulerType().equals(SchedulerType.CAPACITY)) {
+ return;
+ }
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ // start RM
+ MockRM rm1 = new MockRM(conf, memStore);
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ int CONTAINER_MEMORY = 1024;
+ // create app and launch the AM
+ RMApp app0 = rm1.submitApp(CONTAINER_MEMORY);
+ MockAM am0 = MockRM.launchAM(app0, rm1, nm1);
+ nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1,
+ ContainerState.COMPLETE);
+ am0.waitForState(RMAppAttemptState.FAILED);
+ for (int i = 0; i < 4; i++) {
+ am0 = MockRM.launchAM(app0, rm1, nm1);
+ am0.registerAppAttempt();
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ // get scheduler app
+ FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
+ .get(app0.getApplicationId()).getCurrentAppAttempt();
+ // kill app0-attempt
+ cs.killPreemptedContainer(schedulerAppAttempt.getRMContainer(
+ app0.getCurrentAppAttempt().getMasterContainer().getId()));
+ }
+ am0 = MockRM.launchAM(app0, rm1, nm1);
+ am0.registerAppAttempt();
+ rm1.killApp(app0.getApplicationId());
+ rm1.waitForState(app0.getCurrentAppAttempt().getAppAttemptId(),
+ RMAppAttemptState.KILLED);
+
+ MockRM rm2 = null;
+ // start RM2
+ try {
+ rm2 = new MockRM(conf, memStore);
+ rm2.start();
+ Assert.assertTrue("RM start successfully", true);
+ } catch (Exception e) {
+ LOG.debug("Exception on start", e);
+ Assert.fail("RM should start with out any issue");
+ } finally {
+ rm1.stop();
+ }
+ }
+
}
[05/50] [abbrv] hadoop git commit: HDFS-9661. Deadlock in
DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by
ade)
Posted by ka...@apache.org.
HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw (Contributed by ade)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14255786
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14255786
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14255786
Branch: refs/heads/YARN-1011
Commit: 14255786908f991fd2022480fe5575533a3dc7ce
Parents: 57d0a94
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jan 20 21:04:21 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jan 20 21:04:21 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/datanode/fsdataset/impl/FsDatasetImpl.java | 11 +++++++++--
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14255786/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b051f2f..e6074bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2646,6 +2646,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9648. TestStartup.testImageChecksum is broken by HDFS-9569's message
change. (Wei-Chiu Chuang via Yongjun Zhang)
+ HDFS-9661. Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and
+ createRbw (ade via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14255786/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 5350974..cbd4123 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -936,8 +936,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
+ replicaInfo.getVolume().getStorageType());
}
- try (FsVolumeReference volumeRef = volumes.getNextVolume(
- targetStorageType, block.getNumBytes())) {
+ FsVolumeReference volumeRef = null;
+ synchronized (this) {
+ volumeRef = volumes.getNextVolume(targetStorageType, block.getNumBytes());
+ }
+ try {
File oldBlockFile = replicaInfo.getBlockFile();
File oldMetaFile = replicaInfo.getMetaFile();
FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
@@ -956,6 +959,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+ } finally {
+ if (volumeRef != null) {
+ volumeRef.close();
+ }
}
// Replace the old block if any to reschedule the scanning.
[21/50] [abbrv] hadoop git commit: HADOOP-12651. Replace dev-support
with wrappers to Yetus (aw)
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
deleted file mode 100755
index 3c52912..0000000
--- a/dev-support/test-patch.sh
+++ /dev/null
@@ -1,2814 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BUILD_URL is set by Hudson if it is run by patch process
-
-this="${BASH_SOURCE-$0}"
-BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-CWD=$(pwd)
-USER_PARAMS=("$@")
-GLOBALTIMER=$(date +"%s")
-
-## @description Setup the default global variables
-## @audience public
-## @stability stable
-## @replaceable no
-function setup_defaults
-{
- if [[ -z "${MAVEN_HOME:-}" ]]; then
- MVN=mvn
- else
- MVN=${MAVEN_HOME}/bin/mvn
- fi
- # This parameter needs to be kept as an array
- MAVEN_ARGS=()
-
- PROJECT_NAME=hadoop
- HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
- JENKINS=false
- BASEDIR=$(pwd)
- RELOCATE_PATCH_DIR=false
-
- USER_PLUGIN_DIR=""
- LOAD_SYSTEM_PLUGINS=true
-
- FINDBUGS_HOME=${FINDBUGS_HOME:-}
- FINDBUGS_WARNINGS_FAIL_PRECHECK=false
- ECLIPSE_HOME=${ECLIPSE_HOME:-}
- BUILD_NATIVE=${BUILD_NATIVE:-true}
- PATCH_BRANCH=""
- PATCH_BRANCH_DEFAULT="trunk"
- CHANGED_MODULES=""
- USER_MODULE_LIST=""
- OFFLINE=false
- CHANGED_FILES=""
- REEXECED=false
- RESETREPO=false
- ISSUE=""
- ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
- TIMER=$(date +"%s")
- PATCHURL=""
-
- OSTYPE=$(uname -s)
-
- # Solaris needs POSIX, not SVID
- case ${OSTYPE} in
- SunOS)
- PS=${PS:-ps}
- AWK=${AWK:-/usr/xpg4/bin/awk}
- SED=${SED:-/usr/xpg4/bin/sed}
- WGET=${WGET:-wget}
- GIT=${GIT:-git}
- EGREP=${EGREP:-/usr/xpg4/bin/egrep}
- GREP=${GREP:-/usr/xpg4/bin/grep}
- PATCH=${PATCH:-patch}
- DIFF=${DIFF:-/usr/gnu/bin/diff}
- JIRACLI=${JIRA:-jira}
- FILE=${FILE:-file}
- ;;
- *)
- PS=${PS:-ps}
- AWK=${AWK:-awk}
- SED=${SED:-sed}
- WGET=${WGET:-wget}
- GIT=${GIT:-git}
- EGREP=${EGREP:-egrep}
- GREP=${GREP:-grep}
- PATCH=${PATCH:-patch}
- DIFF=${DIFF:-diff}
- JIRACLI=${JIRA:-jira}
- FILE=${FILE:-file}
- ;;
- esac
-
- declare -a JIRA_COMMENT_TABLE
- declare -a JIRA_FOOTER_TABLE
- declare -a JIRA_HEADER
- declare -a JIRA_TEST_TABLE
-
- JFC=0
- JTC=0
- JTT=0
- RESULT=0
-}
-
-## @description Print a message to stderr
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function hadoop_error
-{
- echo "$*" 1>&2
-}
-
-## @description Print a message to stderr if --debug is turned on
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function hadoop_debug
-{
- if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
- echo "[$(date) DEBUG]: $*" 1>&2
- fi
-}
-
-## @description Activate the local timer
-## @audience public
-## @stability stable
-## @replaceable no
-function start_clock
-{
- hadoop_debug "Start clock"
- TIMER=$(date +"%s")
-}
-
-## @description Print the elapsed time in seconds since the start of the local timer
-## @audience public
-## @stability stable
-## @replaceable no
-function stop_clock
-{
- local -r stoptime=$(date +"%s")
- local -r elapsed=$((stoptime-TIMER))
- hadoop_debug "Stop clock"
-
- echo ${elapsed}
-}
-
-## @description Print the elapsed time in seconds since the start of the global timer
-## @audience private
-## @stability stable
-## @replaceable no
-function stop_global_clock
-{
- local -r stoptime=$(date +"%s")
- local -r elapsed=$((stoptime-GLOBALTIMER))
- hadoop_debug "Stop global clock"
-
- echo ${elapsed}
-}
-
-## @description Add time to the local timer
-## @audience public
-## @stability stable
-## @replaceable no
-## @param seconds
-function offset_clock
-{
- ((TIMER=TIMER-$1))
-}
-
-## @description Add to the header of the display
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function add_jira_header
-{
- JIRA_HEADER[${JHC}]="| $* |"
- JHC=$(( JHC+1 ))
-}
-
-## @description Add to the output table. If the first parameter is a number
-## @description that is the vote for that column and calculates the elapsed time
-## @description based upon the last start_clock(). If it the string null, then it is
-## @description a special entry that signifies extra
-## @description content for the final column. The second parameter is the reporting
-## @description subsystem (or test) that is providing the vote. The second parameter
-## @description is always required. The third parameter is any extra verbage that goes
-## @description with that subsystem.
-## @audience public
-## @stability stable
-## @replaceable no
-## @param +1/0/-1/null
-## @param subsystem
-## @param string
-## @return Elapsed time display
-function add_jira_table
-{
- local value=$1
- local subsystem=$2
- shift 2
-
- local color
- local calctime=0
-
- local -r elapsed=$(stop_clock)
-
- if [[ ${elapsed} -lt 0 ]]; then
- calctime="N/A"
- else
- printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
- fi
-
- echo ""
- echo "Elapsed time: ${calctime}"
- echo ""
-
- case ${value} in
- 1|+1)
- value="+1"
- color="green"
- ;;
- -1)
- color="red"
- ;;
- 0)
- color="blue"
- ;;
- null)
- ;;
- esac
-
- if [[ -z ${color} ]]; then
- JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |"
- JTC=$(( JTC+1 ))
- else
- JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |"
- JTC=$(( JTC+1 ))
- fi
-}
-
-## @description Put the final environment information at the bottom
-## @description of the footer table
-## @stability stable
-## @audience private
-## @replaceable yes
-function close_jira_footer
-{
- # shellcheck disable=SC2016
- local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \")
- local -r unamea=$(uname -a)
-
- add_jira_footer "Java" "${javaversion}"
- add_jira_footer "uname" "${unamea}"
-}
-
-## @description Put the final elapsed time at the bottom of the table.
-## @audience private
-## @stability stable
-## @replaceable no
-function close_jira_table
-{
-
- local -r elapsed=$(stop_global_clock)
-
- if [[ ${elapsed} -lt 0 ]]; then
- calctime="N/A"
- else
- printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
- fi
-
- echo ""
- echo "Total Elapsed time: ${calctime}"
- echo ""
-
-
- JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |"
- JTC=$(( JTC+1 ))
-}
-
-## @description Add to the footer of the display. @@BASE@@ will get replaced with the
-## @description correct location for the local filesystem in dev mode or the URL for
-## @description Jenkins mode.
-## @audience public
-## @stability stable
-## @replaceable no
-## @param subsystem
-## @param string
-function add_jira_footer
-{
- local subsystem=$1
- shift 1
-
- JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |"
- JFC=$(( JFC+1 ))
-}
-
-## @description Special table just for unit test failures
-## @audience public
-## @stability stable
-## @replaceable no
-## @param failurereason
-## @param testlist
-function add_jira_test_table
-{
- local failure=$1
- shift 1
-
- JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |"
- JTT=$(( JTT+1 ))
-}
-
-## @description Large display for the user console
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-## @return large chunk of text
-function big_console_header
-{
- local text="$*"
- local spacing=$(( (75+${#text}) /2 ))
- printf "\n\n"
- echo "============================================================================"
- echo "============================================================================"
- printf "%*s\n" ${spacing} "${text}"
- echo "============================================================================"
- echo "============================================================================"
- printf "\n\n"
-}
-
-## @description Remove {color} tags from a string
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-## @return string
-function colorstripper
-{
- local string=$1
- shift 1
-
- local green=""
- local white=""
- local red=""
- local blue=""
-
- echo "${string}" | \
- ${SED} -e "s,{color:red},${red},g" \
- -e "s,{color:green},${green},g" \
- -e "s,{color:blue},${blue},g" \
- -e "s,{color},${white},g"
-}
-
-## @description Find the largest size of a column of an array
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return size
-function findlargest
-{
- local column=$1
- shift
- local a=("$@")
- local sizeofa=${#a[@]}
- local i=0
-
- until [[ ${i} -gt ${sizeofa} ]]; do
- # shellcheck disable=SC2086
- string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| )
- if [[ ${#string} -gt $maxlen ]]; then
- maxlen=${#string}
- fi
- i=$((i+1))
- done
- echo "${maxlen}"
-}
-
-## @description Verify that ${JAVA_HOME} is defined
-## @audience public
-## @stability stable
-## @replaceable no
-## @return 1 - no JAVA_HOME
-## @return 0 - JAVA_HOME defined
-function find_java_home
-{
- start_clock
- if [[ -z ${JAVA_HOME:-} ]]; then
- case $(uname -s) in
- Darwin)
- if [[ -z "${JAVA_HOME}" ]]; then
- if [[ -x /usr/libexec/java_home ]]; then
- JAVA_HOME="$(/usr/libexec/java_home)"
- export JAVA_HOME
- else
- export JAVA_HOME=/Library/Java/Home
- fi
- fi
- ;;
- *)
- ;;
- esac
- fi
-
- if [[ -z ${JAVA_HOME:-} ]]; then
- echo "JAVA_HOME is not defined."
- add_jira_table -1 pre-patch "JAVA_HOME is not defined."
- return 1
- fi
- return 0
-}
-
-## @description Write the contents of a file to jenkins
-## @params filename
-## @stability stable
-## @audience public
-## @returns ${JIRACLI} exit code
-function write_to_jira
-{
- local -r commentfile=${1}
- shift
-
- local retval
-
- if [[ ${OFFLINE} == false
- && ${JENKINS} == true ]]; then
- export USER=hudson
- # shellcheck disable=SC2086
- ${JIRACLI} --comment "$(cat ${commentfile})" \
- -s https://issues.apache.org/jira \
- -a addcomment -u hadoopqa \
- -p "${JIRA_PASSWD}" \
- --issue "${ISSUE}"
- retval=$?
- ${JIRACLI} -s https://issues.apache.org/jira \
- -a logout -u hadoopqa \
- -p "${JIRA_PASSWD}"
- fi
- return ${retval}
-}
-
-## @description Verify that the patch directory is still in working order
-## @description since bad actors on some systems wipe it out. If not,
-## @description recreate it and then exit
-## @audience private
-## @stability evolving
-## @replaceable yes
-## @returns may exit on failure
-function verify_patchdir_still_exists
-{
- local -r commentfile=/tmp/testpatch.$$.${RANDOM}
- local extra=""
-
- if [[ ! -d ${PATCH_DIR} ]]; then
- rm "${commentfile}" 2>/dev/null
-
- echo "(!) The patch artifact directory has been removed! " > "${commentfile}"
- echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}"
- echo
- cat ${commentfile}
- echo
- if [[ ${JENKINS} == true ]]; then
- if [[ -n ${NODE_NAME} ]]; then
- extra=" (node ${NODE_NAME})"
- fi
- echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}"
-
- write_to_jira ${commentfile}
- fi
-
- rm "${commentfile}"
- cleanup_and_exit ${RESULT}
- fi
-}
-
-## @description generate a list of all files and line numbers that
-## @description that were added/changed in the source repo
-## @audience private
-## @stability stable
-## @params filename
-## @replaceable no
-function compute_gitdiff
-{
- local outfile=$1
- local file
- local line
- local startline
- local counter
- local numlines
- local actual
-
- pushd "${BASEDIR}" >/dev/null
- while read line; do
- if [[ ${line} =~ ^\+\+\+ ]]; then
- file="./"$(echo "${line}" | cut -f2- -d/)
- continue
- elif [[ ${line} =~ ^@@ ]]; then
- startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + )
- numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, )
- # if this is empty, then just this line
- # if it is 0, then no lines were added and this part of the patch
- # is strictly a delete
- if [[ ${numlines} == 0 ]]; then
- continue
- elif [[ -z ${numlines} ]]; then
- numlines=1
- fi
- counter=0
- until [[ ${counter} -gt ${numlines} ]]; do
- ((actual=counter+startline))
- echo "${file}:${actual}:" >> "${outfile}"
- ((counter=counter+1))
- done
- fi
- done < <("${GIT}" diff --unified=0 --no-color)
- popd >/dev/null
-}
-
-## @description Print the command to be executing to the screen. Then
-## @description run the command, sending stdout and stderr to the given filename
-## @description This will also ensure that any directories in ${BASEDIR} have
-## @description the exec bit set as a pre-exec step.
-## @audience public
-## @stability stable
-## @param filename
-## @param command
-## @param [..]
-## @replaceable no
-## @returns $?
-function echo_and_redirect
-{
- local logfile=$1
- shift
-
- verify_patchdir_still_exists
-
- find "${BASEDIR}" -type d -exec chmod +x {} \;
- echo "${*} > ${logfile} 2>&1"
- "${@}" > "${logfile}" 2>&1
-}
-
-## @description is PATCH_DIR relative to BASEDIR?
-## @audience public
-## @stability stable
-## @replaceable yes
-## @returns 1 - no, PATCH_DIR
-## @returns 0 - yes, PATCH_DIR - BASEDIR
-function relative_patchdir
-{
- local p=${PATCH_DIR#${BASEDIR}}
-
- if [[ ${#p} -eq ${#PATCH_DIR} ]]; then
- echo ${p}
- return 1
- fi
- p=${p#/}
- echo ${p}
- return 0
-}
-
-
-## @description Print the usage information
-## @audience public
-## @stability stable
-## @replaceable no
-function hadoop_usage
-{
- local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]')
-
- echo "Usage: test-patch.sh [options] patch-file | issue-number | http"
- echo
- echo "Where:"
- echo " patch-file is a local patch file containing the changes to test"
- echo " issue-number is a 'Patch Available' JIRA defect number (e.g. '${up}-9902') to test"
- echo " http is an HTTP address to download the patch file"
- echo
- echo "Options:"
- echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
- echo "--branch=<ref> Forcibly set the branch"
- echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')"
- echo "--build-native=<bool> If true, then build native components (default 'true')"
- echo "--contrib-guide=<url> URL to point new users towards project conventions. (default Hadoop's wiki)"
- echo "--debug If set, then output some extra stuff to stderr"
- echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
- echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
- echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
- echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
- echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
- echo "--offline Avoid connecting to the Internet"
- echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')"
- echo "--plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)"
- echo "--project=<name> The short name for project currently using test-patch (default 'hadoop')"
- echo "--resetrepo Forcibly clean the repo"
- echo "--run-tests Run all relevant tests below the base directory"
- echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d"
- echo "--testlist=<list> Specify which subsystem tests to use (comma delimited)"
- echo "--test-parallel=<bool> Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)"
- echo "--test-threads=<int> Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
-
- echo "Shell binary overrides:"
- echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
- echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')"
- echo "--file-cmd=<cmd> The 'file' command to use (default 'file')"
- echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
- echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
- echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
- echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
- echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
- echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')"
-
- echo
- echo "Jenkins-only options:"
- echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
- echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
- echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
- echo "--jira-password=<pw> The password for the 'jira' command"
- echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
- echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
-}
-
-## @description Interpret the command line parameters
-## @audience private
-## @stability stable
-## @replaceable no
-## @params $@
-## @return May exit on failure
-function parse_args
-{
- local i
- local j
-
- for i in "$@"; do
- case ${i} in
- --awk-cmd=*)
- AWK=${i#*=}
- ;;
- --basedir=*)
- BASEDIR=${i#*=}
- ;;
- --branch=*)
- PATCH_BRANCH=${i#*=}
- ;;
- --branch-default=*)
- PATCH_BRANCH_DEFAULT=${i#*=}
- ;;
- --build-native=*)
- BUILD_NATIVE=${i#*=}
- ;;
- --contrib-guide=*)
- HOW_TO_CONTRIBUTE=${i#*=}
- ;;
- --debug)
- HADOOP_SHELL_SCRIPT_DEBUG=true
- ;;
- --diff-cmd=*)
- DIFF=${i#*=}
- ;;
- --dirty-workspace)
- DIRTY_WORKSPACE=true
- ;;
- --eclipse-home=*)
- ECLIPSE_HOME=${i#*=}
- ;;
- --file-cmd=*)
- FILE=${i#*=}
- ;;
- --findbugs-home=*)
- FINDBUGS_HOME=${i#*=}
- ;;
- --findbugs-strict-precheck)
- FINDBUGS_WARNINGS_FAIL_PRECHECK=true
- ;;
- --git-cmd=*)
- GIT=${i#*=}
- ;;
- --grep-cmd=*)
- GREP=${i#*=}
- ;;
- --help|-help|-h|help|--h|--\?|-\?|\?)
- hadoop_usage
- exit 0
- ;;
- --issue-re=*)
- ISSUE_RE=${i#*=}
- ;;
- --java-home=*)
- JAVA_HOME=${i#*=}
- ;;
- --jenkins)
- JENKINS=true
- TEST_PARALLEL=${TEST_PARALLEL:-true}
- ;;
- --jira-cmd=*)
- JIRACLI=${i#*=}
- ;;
- --jira-password=*)
- JIRA_PASSWD=${i#*=}
- ;;
- --modulelist=*)
- USER_MODULE_LIST=${i#*=}
- USER_MODULE_LIST=${USER_MODULE_LIST//,/ }
- hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}"
- ;;
- --mvn-cmd=*)
- MVN=${i#*=}
- ;;
- --mv-patch-dir)
- RELOCATE_PATCH_DIR=true;
- ;;
- --offline)
- OFFLINE=true
- ;;
- --patch-cmd=*)
- PATCH=${i#*=}
- ;;
- --patch-dir=*)
- USER_PATCH_DIR=${i#*=}
- ;;
- --plugins=*)
- USER_PLUGIN_DIR=${i#*=}
- ;;
- --project=*)
- PROJECT_NAME=${i#*=}
- ;;
- --ps-cmd=*)
- PS=${i#*=}
- ;;
- --reexec)
- REEXECED=true
- start_clock
- add_jira_table 0 reexec "dev-support patch detected."
- ;;
- --resetrepo)
- RESETREPO=true
- ;;
- --run-tests)
- RUN_TESTS=true
- ;;
- --skip-system-plugins)
- LOAD_SYSTEM_PLUGINS=false
- ;;
- --testlist=*)
- testlist=${i#*=}
- testlist=${testlist//,/ }
- for j in ${testlist}; do
- hadoop_debug "Manually adding patch test subsystem ${j}"
- add_test "${j}"
- done
- ;;
- --test-parallel=*)
- TEST_PARALLEL=${i#*=}
- ;;
- --test-threads=*)
- TEST_THREADS=${i#*=}
- ;;
- --wget-cmd=*)
- WGET=${i#*=}
- ;;
- *)
- PATCH_OR_ISSUE=${i}
- ;;
- esac
- done
-
- # if we requested offline, pass that to mvn
- if [[ ${OFFLINE} == "true" ]] ; then
- MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
- fi
-
- # we need absolute dir for ${BASEDIR}
- cd "${CWD}"
- BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
-
- if [[ ${BUILD_NATIVE} == "true" ]] ; then
- NATIVE_PROFILE=-Pnative
- REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
- fi
- if [[ -z "${PATCH_OR_ISSUE}" ]]; then
- hadoop_usage
- exit 1
- fi
- if [[ ${JENKINS} == "true" ]] ; then
- echo "Running in Jenkins mode"
- ISSUE=${PATCH_OR_ISSUE}
- RESETREPO=true
- # shellcheck disable=SC2034
- ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}"
- else
- if [[ ${RESETREPO} == "true" ]] ; then
- echo "Running in destructive (--resetrepo) developer mode"
- else
- echo "Running in developer mode"
- fi
- JENKINS=false
- fi
-
- if [[ -n ${USER_PATCH_DIR} ]]; then
- PATCH_DIR="${USER_PATCH_DIR}"
- else
- PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$
- fi
-
- cd "${CWD}"
- if [[ ! -d ${PATCH_DIR} ]]; then
- mkdir -p "${PATCH_DIR}"
- if [[ $? == 0 ]] ; then
- echo "${PATCH_DIR} has been created"
- else
- echo "Unable to create ${PATCH_DIR}"
- cleanup_and_exit 1
- fi
- fi
-
- # we need absolute dir for PATCH_DIR
- PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
-
- GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt
-
- if [[ ${TEST_PARALLEL} == "true" ]] ; then
- PARALLEL_TESTS_PROFILE=-Pparallel-tests
- if [[ -n ${TEST_THREADS:-} ]]; then
- TESTS_THREAD_COUNT="-DtestsThreadCount=$TEST_THREADS"
- fi
- fi
-}
-
-## @description Locate the pom.xml file for a given directory
-## @audience private
-## @stability stable
-## @replaceable no
-## @return directory containing the pom.xml
-function find_pom_dir
-{
- local dir
-
- dir=$(dirname "$1")
-
- hadoop_debug "Find pom dir for: ${dir}"
-
- while builtin true; do
- if [[ -f "${dir}/pom.xml" ]];then
- echo "${dir}"
- hadoop_debug "Found: ${dir}"
- return
- else
- dir=$(dirname "${dir}")
- fi
- done
-}
-
-## @description List of files that ${PATCH_DIR}/patch modifies
-## @audience private
-## @stability stable
-## @replaceable no
-## @return None; sets ${CHANGED_FILES}
-function find_changed_files
-{
- # get a list of all of the files that have been changed,
- # except for /dev/null (which would be present for new files).
- # Additionally, remove any a/ b/ patterns at the front
- # of the patch filenames and any revision info at the end
- # shellcheck disable=SC2016
- CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \
- | ${SED} \
- -e 's,^....,,' \
- -e 's,^[ab]/,,' \
- | ${GREP} -v /dev/null \
- | ${AWK} '{print $1}' \
- | sort -u)
-}
-
-## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies
-## @audience private
-## @stability stable
-## @replaceable no
-## @return None; sets ${CHANGED_MODULES}
-function find_changed_modules
-{
- # Come up with a list of changed files into ${TMP}
- local pomdirs
- local module
- local pommods
-
- # Now find all the modules that were changed
- for file in ${CHANGED_FILES}; do
- #shellcheck disable=SC2086
- pomdirs="${pomdirs} $(find_pom_dir ${file})"
- done
-
- # Filter out modules without code
- for module in ${pomdirs}; do
- ${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null
- if [[ "$?" != 0 ]]; then
- pommods="${pommods} ${module}"
- fi
- done
-
- #shellcheck disable=SC2086
- CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
-}
-
-## @description git checkout the appropriate branch to test. Additionally, this calls
-## @description 'determine_issue' and 'determine_branch' based upon the context provided
-## @description in ${PATCH_DIR} and in git after checkout.
-## @audience private
-## @stability stable
-## @replaceable no
-## @return 0 on success. May exit on failure.
-function git_checkout
-{
- local currentbranch
- local exemptdir
-
- big_console_header "Confirming git environment"
-
- cd "${BASEDIR}"
- if [[ ! -d .git ]]; then
- hadoop_error "ERROR: ${BASEDIR} is not a git repo."
- cleanup_and_exit 1
- fi
-
- if [[ ${RESETREPO} == "true" ]] ; then
- ${GIT} reset --hard
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git reset is failing"
- cleanup_and_exit 1
- fi
-
- # if PATCH_DIR is in BASEDIR, then we don't want
- # git wiping it out.
- exemptdir=$(relative_patchdir)
- if [[ $? == 1 ]]; then
- ${GIT} clean -xdf
- else
- # we do, however, want it emptied of all _files_.
- # we need to leave _directories_ in case we are in
- # re-exec mode (which places a directory full of stuff in it)
- hadoop_debug "Exempting ${exemptdir} from clean"
- rm "${PATCH_DIR}/*" 2>/dev/null
- ${GIT} clean -xdf -e "${exemptdir}"
- fi
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git clean is failing"
- cleanup_and_exit 1
- fi
-
- ${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}"
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
- cleanup_and_exit 1
- fi
-
- determine_branch
- if [[ ${PATCH_BRANCH} =~ ^git ]]; then
- PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
- fi
-
- # we need to explicitly fetch in case the
- # git ref hasn't been brought in tree yet
- if [[ ${OFFLINE} == false ]]; then
-
- if [[ -f .git/rebase-apply ]]; then
- hadoop_error "ERROR: previous rebase failed. Aborting it."
- ${GIT} rebase --abort
- fi
-
- ${GIT} pull --rebase
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git pull is failing"
- cleanup_and_exit 1
- fi
- fi
- # forcibly checkout this branch or git ref
- ${GIT} checkout --force "${PATCH_BRANCH}"
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
- cleanup_and_exit 1
- fi
-
- # if we've selected a feature branch that has new changes
- # since our last build, we'll need to rebase to see those changes.
- if [[ ${OFFLINE} == false ]]; then
- ${GIT} pull --rebase
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git pull is failing"
- cleanup_and_exit 1
- fi
- fi
-
- else
-
- status=$(${GIT} status --porcelain)
- if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then
- hadoop_error "ERROR: --dirty-workspace option not provided."
- hadoop_error "ERROR: can't run in a workspace that contains the following modifications"
- hadoop_error "${status}"
- cleanup_and_exit 1
- fi
-
- determine_branch
- if [[ ${PATCH_BRANCH} =~ ^git ]]; then
- PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
- fi
-
- currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD)
- if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then
- echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}."
- echo "WARNING: Continuing anyway..."
- PATCH_BRANCH=${currentbranch}
- fi
- fi
-
- determine_issue
-
- GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD)
- # shellcheck disable=SC2034
- VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum}
-
- if [[ "${ISSUE}" == 'Unknown' ]]; then
- echo "Testing patch on ${PATCH_BRANCH}."
- else
- echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
- fi
-
- add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}"
-
- if [[ ! -f ${BASEDIR}/pom.xml ]]; then
- hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting."
- add_jira_table -1 pre-patch "Unsupported build system."
- output_to_jira 1
- cleanup_and_exit 1
- fi
- return 0
-}
-
-## @description Confirm the source environment is compilable
-## @audience private
-## @stability stable
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_without_patch
-{
- local -r mypwd=$(pwd)
-
- big_console_header "Pre-patch ${PATCH_BRANCH} Java verification"
-
- start_clock
-
- verify_needed_test javac
-
- if [[ $? == 1 ]]; then
- echo "Compiling ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
- if [[ $? != 0 ]] ; then
- echo "${PATCH_BRANCH} compilation is broken?"
- add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need javac tests."
- fi
-
- verify_needed_test javadoc
-
- if [[ $? == 1 ]]; then
- echo "Javadoc'ing ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need javadoc tests."
- fi
-
- verify_needed_test site
-
- if [[ $? == 1 ]]; then
- echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need site tests."
- fi
-
- precheck_findbugs
-
- if [[ $? != 0 ]] ; then
- return 1
- fi
-
- add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
- return 0
-}
-
-## @description Confirm the given branch is a member of the list of space
-## @description delimited branches or a git ref
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param branch
-## @param branchlist
-## @return 0 on success
-## @return 1 on failure
-function verify_valid_branch
-{
- local branches=$1
- local check=$2
- local i
-
- # shortcut some common
- # non-resolvable names
- if [[ -z ${check} ]]; then
- return 1
- fi
-
- if [[ ${check} == patch ]]; then
- return 1
- fi
-
- if [[ ${check} =~ ^git ]]; then
- ref=$(echo "${check}" | cut -f2 -dt)
- count=$(echo "${ref}" | wc -c | tr -d ' ')
-
- if [[ ${count} == 8 || ${count} == 41 ]]; then
- return 0
- fi
- return 1
- fi
-
- for i in ${branches}; do
- if [[ "${i}" == "${check}" ]]; then
- return 0
- fi
- done
- return 1
-}
-
-## @description Try to guess the branch being tested using a variety of heuristics
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success, with PATCH_BRANCH updated appropriately
-## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT
-function determine_branch
-{
- local allbranches
- local patchnamechunk
-
- hadoop_debug "Determine branch"
-
- # something has already set this, so move on
- if [[ -n ${PATCH_BRANCH} ]]; then
- return
- fi
-
- pushd "${BASEDIR}" > /dev/null
-
- # developer mode, existing checkout, whatever
- if [[ "${DIRTY_WORKSPACE}" == true ]];then
- PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD)
- echo "dirty workspace mode; applying against existing branch"
- return
- fi
-
- allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g)
-
- for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
- hadoop_debug "Determine branch: starting with ${j}"
- # shellcheck disable=SC2016
- patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}')
-
- # ISSUE.branch.##.patch
- hadoop_debug "Determine branch: ISSUE.branch.##.patch"
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. )
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
-
- # ISSUE-branch-##.patch
- hadoop_debug "Determine branch: ISSUE-branch-##.patch"
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
-
- # ISSUE-##.patch.branch
- hadoop_debug "Determine branch: ISSUE-##.patch.branch"
- # shellcheck disable=SC2016
- PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}')
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
-
- # ISSUE-branch.##.patch
- hadoop_debug "Determine branch: ISSUE-branch.##.patch"
- # shellcheck disable=SC2016
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null)
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
- done
-
- PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
-
- popd >/dev/null
-}
-
-## @description Try to guess the issue being tested using a variety of heuristics
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success, with ISSUE updated appropriately
-## @return 1 on failure, with ISSUE updated to "Unknown"
-function determine_issue
-{
- local patchnamechunk
- local maybeissue
-
- hadoop_debug "Determine issue"
-
- # we can shortcut jenkins
- if [[ ${JENKINS} == true ]]; then
- ISSUE=${PATCH_OR_ISSUE}
- return 0
- fi
-
- # shellcheck disable=SC2016
- patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}')
-
- maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
-
- if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then
- ISSUE=${maybeissue}
- return 0
- fi
-
- ISSUE="Unknown"
- return 1
-}
-
-## @description Add the given test type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-function add_test
-{
- local testname=$1
-
- hadoop_debug "Testing against ${testname}"
-
- if [[ -z ${NEEDED_TESTS} ]]; then
- hadoop_debug "Setting tests to ${testname}"
- NEEDED_TESTS=${testname}
- elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then
- hadoop_debug "Adding ${testname}"
- NEEDED_TESTS="${NEEDED_TESTS} ${testname}"
- fi
-}
-
-## @description Verify if a given test was requested
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-## @return 1 = yes
-## @return 0 = no
-function verify_needed_test
-{
- local i=$1
-
- if [[ ${NEEDED_TESTS} =~ $i ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Use some heuristics to determine which long running
-## @description tests to run
-## @audience private
-## @stability stable
-## @replaceable no
-function determine_needed_tests
-{
- local i
-
- for i in ${CHANGED_FILES}; do
- if [[ ${i} =~ src/main/webapp ]]; then
- hadoop_debug "tests/webapp: ${i}"
- elif [[ ${i} =~ \.sh
- || ${i} =~ \.cmd
- ]]; then
- hadoop_debug "tests/shell: ${i}"
- elif [[ ${i} =~ \.md$
- || ${i} =~ \.md\.vm$
- || ${i} =~ src/site
- || ${i} =~ src/main/docs
- ]]; then
- hadoop_debug "tests/site: ${i}"
- add_test site
- elif [[ ${i} =~ \.c$
- || ${i} =~ \.cc$
- || ${i} =~ \.h$
- || ${i} =~ \.hh$
- || ${i} =~ \.proto$
- || ${i} =~ src/test
- || ${i} =~ \.cmake$
- || ${i} =~ CMakeLists.txt
- ]]; then
- hadoop_debug "tests/units: ${i}"
- add_test javac
- add_test unit
- elif [[ ${i} =~ pom.xml$
- || ${i} =~ \.java$
- || ${i} =~ src/main
- ]]; then
- hadoop_debug "tests/javadoc+units: ${i}"
- add_test javadoc
- add_test javac
- add_test unit
- fi
-
- if [[ ${i} =~ \.java$ ]]; then
- add_test findbugs
- fi
-
- for plugin in ${PLUGINS}; do
- if declare -f ${plugin}_filefilter >/dev/null 2>&1; then
- "${plugin}_filefilter" "${i}"
- fi
- done
- done
-
- add_jira_footer "Optional Tests" "${NEEDED_TESTS}"
-}
-
-## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the
-## @description necessary work to place it into ${PATCH_DIR}/patch.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure, may exit
-function locate_patch
-{
- local notSureIfPatch=false
- hadoop_debug "locate patch"
-
- if [[ -f ${PATCH_OR_ISSUE} ]]; then
- PATCH_FILE="${PATCH_OR_ISSUE}"
- else
- if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then
- echo "Patch is being downloaded at $(date) from"
- PATCHURL="${PATCH_OR_ISSUE}"
- else
- ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
-
- if [[ $? != 0 ]];then
- hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference."
- cleanup_and_exit 1
- fi
-
- if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
- if [[ ${JENKINS} == true ]]; then
- hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- cleanup_and_exit 1
- else
- hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- fi
- fi
-
- relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*')
- PATCHURL="http://issues.apache.org${relativePatchURL}"
- if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
- notSureIfPatch=true
- fi
- patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*')
- echo "${ISSUE} patch is being downloaded at $(date) from"
- fi
- echo "${PATCHURL}"
- add_jira_footer "Patch URL" "${PATCHURL}"
- ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
- if [[ $? != 0 ]];then
- hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
- cleanup_and_exit 1
- fi
- PATCH_FILE="${PATCH_DIR}/patch"
- fi
-
- if [[ ! -f "${PATCH_DIR}/patch" ]]; then
- cp "${PATCH_FILE}" "${PATCH_DIR}/patch"
- if [[ $? == 0 ]] ; then
- echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}"
- else
- hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
- cleanup_and_exit 1
- fi
- fi
- if [[ ${notSureIfPatch} == "true" ]]; then
- guess_patch_file "${PATCH_DIR}/patch"
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: ${PATCHURL} is not a patch file."
- cleanup_and_exit 1
- else
- hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
- add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
- fi
- fi
-}
-
-## @description Given a possible patch file, guess if it's a patch file without using smart-apply-patch
-## @audience private
-## @stability evolving
-## @param path to patch file to test
-## @return 0 we think it's a patch file
-## @return 1 we think it's not a patch file
-function guess_patch_file
-{
- local patch=$1
- local fileOutput
-
- hadoop_debug "Trying to guess is ${patch} is a patch file."
- fileOutput=$("${FILE}" "${patch}")
- if [[ $fileOutput =~ \ diff\ ]]; then
- hadoop_debug "file magic says it's a diff."
- return 0
- fi
- fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
- if [[ $? == 0 ]]; then
- hadoop_debug "first line looks like a patch file."
- return 0
- fi
- return 1
-}
-
-## @description Given ${PATCH_DIR}/patch, verify the patch is good using ${BINDIR}/smart-apply-patch.sh
-## @description in dryrun mode.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function verify_patch_file
-{
- # Before building, check to make sure that the patch is valid
- export PATCH
-
- "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- add_jira_table -1 patch "The patch command could not apply the patch during dryrun."
- return 1
- else
- return 0
- fi
-}
-
-## @description Given ${PATCH_DIR}/patch, apply the patch using ${BINDIR}/smart-apply-patch.sh
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return exit on failure
-function apply_patch_file
-{
- big_console_header "Applying patch"
-
- export PATCH
- "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch"
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- ((RESULT = RESULT + 1))
- add_jira_table -1 patch "The patch command could not apply the patch."
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
- return 0
-}
-
-
-## @description If this actually patches the files used for the QA process
-## @description under dev-support and its subdirectories, then
-## @description run with the patched version for the test.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return none; otherwise relaunches
-function check_reexec
-{
- local commentfile=${PATCH_DIR}/tp.${RANDOM}
-
- if [[ ${REEXECED} == true ]]; then
- big_console_header "Re-exec mode detected. Continuing."
- return
- fi
-
- if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch
- && ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then
- return
- fi
-
- big_console_header "dev-support patch detected"
-
- if [[ ${RESETREPO} == false ]]; then
- ((RESULT = RESULT + 1))
- hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :("
- add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory."
- return
- fi
-
- printf "\n\nRe-executing against patched versions to test.\n\n"
-
- apply_patch_file
-
- if [[ ${JENKINS} == true ]]; then
-
- rm "${commentfile}" 2>/dev/null
-
- echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}"
- echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}"
- echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}"
-
- write_to_jira "${commentfile}"
- rm "${commentfile}"
- fi
-
- cd "${CWD}"
- mkdir -p "${PATCH_DIR}/dev-support-test"
- (cd "${BINDIR}"; tar cpf - . ) \
- | (cd "${PATCH_DIR}/dev-support-test"; tar xpf - )
-
- big_console_header "exec'ing test-patch.sh now..."
-
- exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
- --reexec \
- --branch="${PATCH_BRANCH}" \
- --patch-dir="${PATCH_DIR}" \
- "${USER_PARAMS[@]}"
-}
-
-## @description Check the current directory for @author tags
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_author
-{
- local authorTags
-
- big_console_header "Checking there are no @author tags in the patch."
-
- start_clock
-
- if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then
- add_jira_table 0 @author "Skipping @author checks as test-patch has been patched."
- return 0
- fi
-
- authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch")
- echo "There appear to be ${authorTags} @author tags in the patch."
- if [[ ${authorTags} != 0 ]] ; then
- add_jira_table -1 @author \
- "The patch appears to contain ${authorTags} @author tags which the Hadoop" \
- " community has agreed to not allow in code contributions."
- return 1
- fi
- add_jira_table +1 @author "The patch does not contain any @author tags."
- return 0
-}
-
-## @description Check the patch file for changed/new tests
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_modified_unittests
-{
- local testReferences=0
- local i
-
- verify_needed_test unit
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "Checking there are new or changed tests in the patch."
-
- start_clock
-
- for i in ${CHANGED_FILES}; do
- if [[ ${i} =~ /test/ ]]; then
- ((testReferences=testReferences + 1))
- fi
- done
-
- echo "There appear to be ${testReferences} test file(s) referenced in the patch."
- if [[ ${testReferences} == 0 ]] ; then
- add_jira_table -1 "tests included" \
- "The patch doesn't appear to include any new or modified tests. " \
- "Please justify why no new tests are needed for this patch." \
- "Also please list what manual steps were performed to verify this patch."
- return 1
- fi
- add_jira_table +1 "tests included" \
- "The patch appears to include ${testReferences} new or modified test files."
- return 0
-}
-
-## @description Helper for check_javadoc
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javadoc_warns
-{
- local warningfile=$1
-
- #shellcheck disable=SC2016,SC2046
- return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}')
-}
-
-## @description Count and compare the number of JavaDoc warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_javadoc
-{
- local numBranchJavadocWarnings
- local numPatchJavadocWarnings
-
- verify_needed_test javadoc
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need javadoc checks."
- return 0
- fi
-
- big_console_header "Determining number of patched javadoc warnings"
-
- start_clock
-
- if [[ -d hadoop-project ]]; then
- (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
- fi
- if [[ -d hadoop-common-project/hadoop-annotations ]]; then
- (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
- fi
- echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
- count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
- numBranchJavadocWarnings=$?
- count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
- numPatchJavadocWarnings=$?
-
- echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch."
- if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then
- if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then
-
- ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt"
- ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
- ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \
- "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \
- > "${PATCH_DIR}/diffJavadocWarnings.txt"
- rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
-
- add_jira_table -1 javadoc "The applied patch generated "\
- "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \
- " additional warning messages."
- add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt"
- return 1
- fi
- fi
- add_jira_table +1 javadoc "There were no new javadoc warning messages."
- return 0
-}
-
-## @description Make sure site still compiles
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_site
-{
- local -r mypwd=$(pwd)
-
- verify_needed_test site
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need site checks."
- return 0
- fi
-
- big_console_header "Determining if patched site still builds"
-
- start_clock
-
- echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Site compilation is broken"
- add_jira_table -1 site "Site compilation is broken."
- add_jira_footer site "@@BASE@@/patchSiteWarnings.txt"
- return 1
- fi
- add_jira_table +1 site "Site still builds."
- return 0
-}
-
-## @description Helper for check_javac
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javac_warns
-{
- local warningfile=$1
- #shellcheck disable=SC2016,SC2046
- return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}")
-}
-
-## @description Count and compare the number of javac warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_javac
-{
- local branchJavacWarnings
- local patchJavacWarnings
-
- verify_needed_test javac
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need javac checks."
- return 0
- fi
-
- big_console_header "Determining number of patched javac warnings."
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
- if [[ $? != 0 ]] ; then
- add_jira_table -1 javac "The patch appears to cause the build to fail."
- return 2
- fi
- ### Compare ${PATCH_BRANCH} and patch javac warning numbers
- if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then
- ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
- ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
-
- count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
- branchJavacWarnings=$?
- count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
- patchJavacWarnings=$?
-
- echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch."
- if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then
- if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then
-
- ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \
- "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \
- > "${PATCH_DIR}/diffJavacWarnings.txt"
-
- add_jira_table -1 javac "The applied patch generated "\
- "$((patchJavacWarnings-branchJavacWarnings))" \
- " additional warning messages."
-
- add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt"
-
- return 1
- fi
- fi
- fi
-
- add_jira_table +1 javac "There were no new javac warning messages."
- return 0
-}
-
-## @description Verify all files have an Apache License
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_apachelicense
-{
- big_console_header "Determining number of patched release audit warnings."
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
- #shellcheck disable=SC2038
- find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
-
- ### Compare ${PATCH_BRANCH} and patch release audit warning numbers
- if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then
- patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt")
- echo ""
- echo ""
- echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch."
- if [[ ${patchReleaseAuditWarnings} != "" ]] ; then
- if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then
- add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings."
-
- ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \
- > "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
- echo "Lines that start with ????? in the release audit "\
- "report indicate files that do not have an Apache license header." \
- >> "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
- add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt"
-
- return 1
- fi
- fi
- fi
- add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings."
- return 0
-}
-
-## @description Verify mvn install works
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_mvn_install
-{
- local retval
-
- verify_needed_test javadoc
- retval=$?
-
- verify_needed_test javac
- ((retval = retval + $? ))
-
- if [[ ${retval} == 0 ]]; then
- echo "This patch does not appear to need mvn install checks."
- return 0
- fi
-
- big_console_header "Installing all of the jars"
-
- start_clock
- echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
- retval=$?
- if [[ ${retval} != 0 ]]; then
- add_jira_table -1 install "The patch causes mvn install to fail."
- else
- add_jira_table +1 install "mvn install still works."
- fi
- return ${retval}
-}
-
-## @description are the needed bits for findbugs present?
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 findbugs will work for our use
-## @return 1 findbugs is missing some component
-function findbugs_is_installed
-{
- if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
- printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
- add_jira_table -1 findbugs "Findbugs is not installed."
- return 1
- fi
- return 0
-}
-
-## @description Run the maven findbugs plugin and record found issues in a bug database
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function findbugs_mvnrunner
-{
- local name=$1
- local logfile=$2
- local warnings_file=$3
-
- echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
- "-D${PROJECT_NAME}PatchProcess" < /dev/null
- if [[ $? != 0 ]]; then
- return 1
- fi
- cp target/findbugsXml.xml "${warnings_file}.xml"
-
- "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
- "${warnings_file}.xml" "${warnings_file}.xml"
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
- "${warnings_file}.html"
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- return 0
-}
-
-## @description Track pre-existing findbugs warnings
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_findbugs
-{
- local -r mypwd=$(pwd)
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
- local findbugs_version
- local rc=0
- local module_findbugs_warnings
- local findbugs_warnings=0
-
- verify_needed_test findbugs
-
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need findbugs tests."
- return 0
- fi
-
- echo "findbugs baseline for ${mypwd}"
-
- findbugs_is_installed
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running findbugs in ${module}"
- module_suffix=$(basename "${module}")
- findbugs_mvnrunner "${PATCH_BRANCH}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
- (( rc = rc + $? ))
-
- if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
- #shellcheck disable=SC2016
- module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
- "${PATCH_BRANCH}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
- | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
-
- if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
- add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
- fi
- fi
- popd >/dev/null
- done
-
- #shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
-
- if [[ ${rc} -ne 0 ]]; then
- echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
- add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
- return 1
- fi
-
- if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
- ${findbugs_warnings} -gt 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
-
- return 0
-}
-
-## @description Verify patch does not trigger any findbugs warnings
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_findbugs
-{
- local rc=0
- local module
- local modules=${CHANGED_MODULES}
- local module_suffix
- local combined_xml
- local newBugs
- local new_findbugs_warnings
- local new_findbugs_fixed_warnings
- local findbugs_warnings=0
- local findbugs_fixed_warnings=0
- local line
- local firstpart
- local secondpart
- local findbugs_version
-
- verify_needed_test findbugs
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "Determining number of patched Findbugs warnings."
-
- start_clock
-
- findbugs_is_installed
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running findbugs in ${module}"
- module_suffix=$(basename "${module}")
-
- findbugs_mvnrunner patch \
- "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch findbugs compilation is broken."
- add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
- continue
- fi
-
- combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
- newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
- "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
- -output "${combined_xml}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- #shellcheck disable=SC2016
- new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
- "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
- #shellcheck disable=SC2016
- new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
- "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
- findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
- findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
-
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
- "${newBugs}.html"
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
- populate_test_table FindBugs "module:${module_suffix}"
- while read line; do
- firstpart=$(echo "${line}" | cut -f2 -d:)
- secondpart=$(echo "${line}" | cut -f9- -d' ')
- add_jira_test_table "" "${firstpart}:${secondpart}"
- done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
-
- add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
- fi
-
- popd >/dev/null
- done
-
- #shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
-
- if [[ ${findbugs_warnings} -gt 0 ]] ; then
- add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
-
- if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
- else
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
- fi
- return 0
-}
-
-## @description Make sure Maven's eclipse generation works.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_mvn_eclipse
-{
- big_console_header "Running mvn eclipse:eclipse."
-
- verify_needed_test javac
- if [[ $? == 0 ]]; then
- echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse"
- return 0
- fi
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
- return 1
- fi
- add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse."
- return 0
-}
-
-## @description Utility to push many tests into the failure list
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param testdesc
-## @param testlist
-function populate_test_table
-{
- local reason=$1
- shift
- local first=""
- local i
-
- for i in "$@"; do
- if [[ -z "${first}" ]]; then
- add_jira_test_table "${reason}" "${i}"
- first="${reason}"
- else
- add_jira_test_table " " "${i}"
- fi
- done
-}
-
-## @description Run and verify the output of the appropriate unit tests
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_unittests
-{
- verify_needed_test unit
-
- if [[ $? == 0 ]]; then
- echo "Existing unit tests do not test patched files. Skipping."
- return 0
- fi
-
- big_console_header "Running unit tests"
-
- start_clock
-
- local failed_tests=""
- local modules=${CHANGED_MODULES}
- local building_common=0
- local hdfs_modules
- local ordered_modules=""
- local failed_test_builds=""
- local test_timeouts=""
- local test_logfile
- local test_build_result
- local module_test_timeouts=""
- local result
- local totalresult=0
- local module_prefix
-
- #
- # If we are building hadoop-hdfs-project, we must build the native component
- # of hadoop-common-project first. In order to accomplish this, we move the
- # hadoop-hdfs subprojects to the end of the list so that common will come
- # first.
- #
- # Of course, we may not be building hadoop-common at all-- in this case, we
- # explicitly insert a mvn compile -Pnative of common, to ensure that the
- # native libraries show up where we need them.
- #
-
- for module in ${modules}; do
- if [[ ${module} == hadoop-hdfs-project* ]]; then
- hdfs_modules="${hdfs_modules} ${module}"
- elif [[ ${module} == hadoop-common-project* ]]; then
- ordered_modules="${ordered_modules} ${module}"
- building_common=1
- else
- ordered_modules="${ordered_modules} ${module}"
- fi
- done
-
- if [[ -n "${hdfs_modules}" ]]; then
- ordered_modules="${ordered_modules} ${hdfs_modules}"
- if [[ ${building_common} -eq 0 ]]; then
- echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
- echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
- if [[ $? != 0 ]]; then
- add_jira_table -1 "native" "Failed to build the native portion " \
- "of hadoop-common prior to running the unit tests in ${ordered_modules}"
- return 1
- else
- add_jira_table +1 "native" "Pre-build of native portion"
- fi
- fi
- fi
-
- for module in ${ordered_modules}; do
- result=0
- start_clock
- pushd "${module}" >/dev/null
- module_suffix=$(basename "${module}")
- module_prefix=$(echo "${module}" | cut -f2 -d- )
-
- test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
- echo " Running tests in ${module_suffix}"
- # shellcheck disable=2086
- echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess
- test_build_result=$?
-
- add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"
-
- # shellcheck disable=2016
- module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${test_logfile}")
- if [[ -n "${module_test_timeouts}" ]] ; then
- test_timeouts="${test_timeouts} ${module_test_timeouts}"
- result=1
- fi
-
- #shellcheck disable=SC2026,SC2038,SC2016
- module_failed_tests=$(find . -name 'TEST*.xml'\
- | xargs "${GREP}" -l -E "<failure|<error"\
- | ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}')
-
- if [[ -n "${module_failed_tests}" ]] ; then
- failed_tests="${failed_tests} ${module_failed_tests}"
- result=1
- fi
- if [[ ${test_build_result} != 0 && -z "${module_failed_tests}" && -z "${module_test_timeouts}" ]] ; then
- failed_test_builds="${failed_test_builds} ${module_suffix}"
- result=1
- fi
- popd >/dev/null
-
- if [[ $result == 1 ]]; then
- add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}."
- else
- add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}."
- fi
-
- ((totalresult = totalresult + result))
- done
-
- if [[ -n "${failed_tests}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Failed unit tests" ${failed_tests}
- fi
- if [[ -n "${test_timeouts}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Timed out tests" ${test_timeouts}
- fi
- if [[ -n "${failed_test_builds}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Failed build" ${failed_test_builds}
- fi
-
- if [[ ${JENKINS} == true ]]; then
- add_jira_footer "Test Results" "${BUILD_URL}testReport/"
- fi
-
- if [[ ${totalresult} -gt 0 ]]; then
- return 1
- else
- return 0
- fi
-}
-
-## @description Print out the finished details on the console
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-## @return 0 on success
-## @return 1 on failure
-function output_to_console
-{
- local result=$1
- shift
- local i
- local ourstring
- local vote
- local subs
- local ela
- local comment
- local commentfile1="${PATCH_DIR}/comment.1"
- local commentfile2="${PATCH_DIR}/comment.2"
- local normaltop
- local line
- local seccoladj=0
- local spcfx=${PATCH_DIR}/spcl.txt
-
- if [[ ${result} == 0 ]]; then
- if [[ ${JENKINS} == false ]]; then
- {
- printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
- printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
- printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
- printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
- printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
- printf "ICB8bXwgICB8bXwgIAo"
- } > "${spcfx}"
- fi
- printf "\n\n+1 overall\n\n"
- else
- if [[ ${JENKINS} == false ]]; then
- {
- printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
- printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
- printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
- printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
- printf "ICAgICAgICAgICAgICAgICAK"
- } > "${spcfx}"
- fi
- printf "\n\n-1 overall\n\n"
- fi
-
- if [[ -f ${spcfx} ]]; then
- if which base64 >/dev/null 2>&1; then
- base64 --decode "${spcfx}" 2>/dev/null
- elif which openssl >/dev/null 2>&1; then
- openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
- fi
- echo
- echo
- rm "${spcfx}"
- fi
-
- seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}")
- if [[ ${seccoladj} -lt 10 ]]; then
- seccoladj=10
- fi
-
- seccoladj=$((seccoladj + 2 ))
- i=0
- until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
- printf "%s\n" "${JIRA_HEADER[${i}]}"
- ((i=i+1))
- done
-
- printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
- echo "============================================================================"
- i=0
- until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
- ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- vote=$(colorstripper "${vote}")
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- ela=$(echo "${ourstring}" | cut -f4 -d\|)
- comment=$(echo "${ourstring}" | cut -f5 -d\|)
-
- echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
- normaltop=$(head -1 "${commentfile1}")
- ${SED} -e '1d' "${commentfile1}" > "${commentfile2}"
-
- printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
- "${subs}" "${ela}" "${normaltop}"
- while read line; do
- printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}"
- done < "${commentfile2}"
-
- ((i=i+1))
- rm "${commentfile2}" "${commentfile1}" 2>/dev/null
- done
-
- if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
- seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}")
- printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
- i=0
- until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
- ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
- ((i=i+1))
- done
- fi
-
- printf "\n\n|| Subsystem || Report/Notes ||\n"
- echo "============================================================================"
- i=0
-
- until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
- printf "%s\n" "${comment}"
- ((i=i+1))
- done
-}
-
-## @description Print out the finished details to the JIRA issue
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-function output_to_jira
-{
- local result=$1
- local i
- local commentfile=${PATCH_DIR}/commentfile
- local comment
-
- rm "${commentfile}" 2>/dev/null
-
- if [[ ${JENKINS} != "true" ]] ; then
- return 0
- fi
-
- big_console_header "Adding comment to JIRA"
-
- add_jira_footer "Console output" "${BUILD_URL}console"
-
- if [[ ${result} == 0 ]]; then
- add_jira_header "(/) *{color:green}+1 overall{color}*"
- else
- add_jira_header "(x) *{color:red}-1 overall{color}*"
- fi
-
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- i=0
- until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
- printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}"
-
- i=0
- until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
- printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
-
-
- if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Reason || Tests ||" >> "${commentfile}"
- i=0
- until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
- printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
- fi
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Subsystem || Report/Notes ||" >> "${commentfile}"
- i=0
- until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g")
- printf "%s\n" "${comment}" >> "${commentfile}"
- ((i=i+1))
- done
-
- printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
-
- write_to_jira "${commentfile}"
-}
-
-## @description Clean the filesystem as appropriate and then exit
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-function cleanup_and_exit
-{
- local result=$1
-
- if [[ ${JENKINS} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \
- -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
- # if PATCH_DIR is already inside BASEDIR, then
- # there is no need to move it since we assume that
- # Jenkins or whatever already knows where it is at
- # since it told us to put it there!
- relative_patchdir >/dev/null
- if [[ $? == 1 ]]; then
- hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
- mv "${PATCH_DIR}" "${BASEDIR}"
- fi
- fi
- big_console_header "Finished build."
-
- # shellcheck disable=SC2086
- exit ${result}
-}
-
-## @description Driver to execute _postcheckout routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function postcheckout
-{
- local routine
- local plugin
-
- for routine in find_java_home verify_patch_file
- do
- verify_patchdir_still_exists
-
- hadoop_debug "Running ${routine}"
- ${routine}
-
- (( RESULT = RESULT + $? ))
- if [[ ${RESULT} != 0 ]] ; then
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
-
- if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then
-
- hadoop_debug "Running ${plugin}_postcheckout"
- #shellcheck disable=SC2086
- ${plugin}_postcheckout
-
- (( RESULT = RESULT + $? ))
- if [[ ${RESULT} != 0 ]] ; then
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
- fi
- done
-}
-
-## @description Driver to execute _preapply routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function preapply
-{
- local routine
- local plugin
-
- for routine in precheck_without_patch check_author \
- check_modified_unittests
- do
- verify_patchdir_still_exists
-
- hadoop_debug "Running ${routine}"
- ${routine}
-
- (( RESULT = RESULT + $? ))
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
-
- if declare -f ${plugin}_preapply >/dev/null 2>&1; then
-
- hadoop_debug "Running ${plugin}_preapply"
- #shellcheck disable=SC2086
- ${plugin}_preapply
-
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Driver to execute _postapply routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function postapply
-{
- local routine
- local plugin
- local retval
-
- compute_gitdiff "${GITDIFFLINES}"
-
- check_javac
- retval=$?
- if [[ ${retval} -gt 1 ]] ; then
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
-
- ((RESULT = RESULT + retval))
-
- for routine in check_javadoc check_apachelicense check_site
- do
- verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
- $routine
-
- (( RESULT = RESULT + $? ))
-
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_postapply >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postapply"
- #shellcheck disable=SC2086
- ${plugin}_postapply
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Driver to execute _postinstall routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function postinstall
-{
- local routine
- local plugin
-
- for routine in check_mvn_eclipse check_findbugs
- do
- verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
- ${routine}
- (( RESULT = RESULT + $? ))
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_postinstall >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postinstall"
- #shellcheck disable=SC2086
- ${plugin}_postinstall
- (( RESULT = RESULT + $? ))
- fi
- done
-
-}
-
-## @description Driver to execute _tests routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function runtests
-{
- local plugin
-
- ### Run tests for Jenkins or if explictly asked for by a developer
- if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then
-
- verify_patchdir_still_exists
- check_unittests
-
- (( RESULT = RESULT + $? ))
- fi
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_tests >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_tests"
- #shellcheck disable=SC2086
- ${plugin}_tests
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Import content from test-patch.d and optionally
-## @description from user provided plugin directory
-## @audience private
-## @stability evolving
-## @replaceable no
-function importplugins
-{
- local i
- local files=()
-
- if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
- if [[ -d "${BINDIR}/test-patch.d" ]]; then
- files=(${BINDIR}/test-patch.d/*.sh)
- fi
- fi
-
- if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
- hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
- files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
- fi
-
- for i in "${files[@]}"; do
- hadoop_debug "Importing ${i}"
- . "${i}"
- done
-}
-
-## @description Register test-patch.d plugins
-## @audience public
-## @stability stable
-## @replaceable no
-function add_plugin
-{
- PLUGINS="${PLUGINS} $1"
-}
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-big_console_header "Bootstrapping test harness"
-
-setup_defaults
-
-parse_args "$@"
-
-importplugins
-
-locate_patch
-
-find_changed_files
-
-determine_needed_tests
-
-# from here on out, we'll be in ${BASEDIR} for cwd
-# routines need to pushd/popd if they change.
-git_checkout
-RESULT=$?
-if [[ ${JENKINS} == "true" ]] ; then
- if [[ ${RESULT} != 0 ]] ; then
- exit 100
- fi
-fi
-
-check_reexec
-
-postcheckout
-
-find_changed_modules
-
-preapply
-
-apply_patch_file
-
-postapply
-
-check_mvn_install
-
-postinstall
-
-runtests
-
-close_jira_footer
-
-close_jira_table
-
-output_to_console ${RESULT}
-output_to_jira ${RESULT}
-cleanup_and_exit ${RESULT}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 108834b..68ad350 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -513,10 +513,9 @@
<goal>exec</goal>
</goals>
<configuration>
- <executable>python</executable>
+ <executable>${basedir}/../../dev-support/bin/shelldocs</executable>
<workingDirectory>src/site/markdown</workingDirectory>
<arguments>
- <argument>${basedir}/../../dev-support/shelldocs.py</argument>
<argument>--skipprnorep</argument>
<argument>--output</argument>
<argument>${basedir}/src/site/markdown/UnixShellAPI.md</argument>
@@ -982,11 +981,10 @@
<goal>exec</goal>
</goals>
<configuration>
- <executable>python</executable>
+ <executable>${basedir}/../../dev-support/bin/releasedocmaker</executable>
<workingDirectory>src/site/markdown/release/</workingDirectory>
<requiresOnline>true</requiresOnline>
<arguments>
- <argument>${basedir}/../../dev-support/releasedocmaker.py</argument>
<argument>--index</argument>
<argument>--license</argument>
<argument>--project</argument>
[23/50] [abbrv] hadoop git commit: HADOOP-12651. Replace dev-support
with wrappers to Yetus (aw)
Posted by ka...@apache.org.
HADOOP-12651. Replace dev-support with wrappers to Yetus (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a867355
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a867355
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a867355
Branch: refs/heads/YARN-1011
Commit: 2a867355dfce880bb82257508862ec26bc7f16b9
Parents: 8cecad2
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jan 21 13:03:34 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jan 21 13:03:34 2016 -0800
----------------------------------------------------------------------
dev-support/README.md | 57 +++++++++++
dev-support/bin/releasedocmaker | 18 ++++
dev-support/bin/shelldocs | 18 ++++
dev-support/bin/smart-apply-patch | 18 ++++
dev-support/bin/test-patch | 18 ++++
dev-support/bin/yetus-wrapper | 175 +++++++++++++++++++++++++++++++++
6 files changed, 304 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a867355/dev-support/README.md
----------------------------------------------------------------------
diff --git a/dev-support/README.md b/dev-support/README.md
new file mode 100644
index 0000000..b2e3476
--- /dev/null
+++ b/dev-support/README.md
@@ -0,0 +1,57 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+This directory contains tools to help in the development and release of Apache Hadoop.
+
+* bin
+
+ * releasedocmaker
+
+ Build release notes for a given Hadoop project or subproject. This is called from Maven when -Preleasedocs is used. See BUILDING.txt for more information.
+
+ * shelldocs
+
+ Create documentation for the Unix Shell API. This is called from Maven when -Pshelldocs is used.
+
+ * smart-apply-patch
+
+ Intelligently apply a patch file to a source tree.
+
+ * test-patch
+
+ Test a patch against a source tree.
+
+* create-release.sh
+
+ Helps REs create a release of Apache Hadoop for distribution.
+
+* determine-flaky-tests-hadoop.py
+
+ Given a jenkins test job, this script examines all runs of the job done within specified period of time (number of days prior to the execution time of this script), and reports all failed tests.
+
+* docker
+
+ Various helpers for the start-build-env.sh script, including the Dockerfile itself. See parent BUILDING.txt for more information.
+
+* findHangingTest.sh
+
+ Finds hanging test from Jenkins build output.
+
+
+Previously, the scripts test-patch.sh, smart-apply-patch.sh, releasedocmaker.py, and shelldocs.py were in this directory. They have been moved to the Apache Yetus project (https://yetus.apache.org). These scripts have been replaced with wrapper scripts located in the bin directory. Command line options are generally different than the previous versions that shipped with older versions of Apache Hadoop.
+
+The wrapper scripts will download, verify (if GPG is installed), and cache a local copy of Apache Yetus in the hadoop/patchprocess directory. The version that is used may be overridden by setting the HADOOP\_YETUS\_VERSION environment variable. The cache directory may be overwritten by setting the HADOOP\_PATCHPROCESS directory. If a local version of Apache Yetus is already installed, it may be used instead by setting the YETUS\_HOME environment variable to point to that directory.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a867355/dev-support/bin/releasedocmaker
----------------------------------------------------------------------
diff --git a/dev-support/bin/releasedocmaker b/dev-support/bin/releasedocmaker
new file mode 100755
index 0000000..16e4c7e
--- /dev/null
+++ b/dev-support/bin/releasedocmaker
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
+exec "${BINDIR}/yetus-wrapper" releasedocmaker "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a867355/dev-support/bin/shelldocs
----------------------------------------------------------------------
diff --git a/dev-support/bin/shelldocs b/dev-support/bin/shelldocs
new file mode 100755
index 0000000..18eda4d
--- /dev/null
+++ b/dev-support/bin/shelldocs
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
+exec "${BINDIR}/yetus-wrapper" shelldocs "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a867355/dev-support/bin/smart-apply-patch
----------------------------------------------------------------------
diff --git a/dev-support/bin/smart-apply-patch b/dev-support/bin/smart-apply-patch
new file mode 100755
index 0000000..3fd469f
--- /dev/null
+++ b/dev-support/bin/smart-apply-patch
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
+exec "${BINDIR}/yetus-wrapper" smart-apply-patch --project=hadoop "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a867355/dev-support/bin/test-patch
----------------------------------------------------------------------
diff --git a/dev-support/bin/test-patch b/dev-support/bin/test-patch
new file mode 100755
index 0000000..8ff8119
--- /dev/null
+++ b/dev-support/bin/test-patch
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
+exec "${BINDIR}/yetus-wrapper" test-patch --project=hadoop --skip-dir=dev-support "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a867355/dev-support/bin/yetus-wrapper
----------------------------------------------------------------------
diff --git a/dev-support/bin/yetus-wrapper b/dev-support/bin/yetus-wrapper
new file mode 100755
index 0000000..37082d8
--- /dev/null
+++ b/dev-support/bin/yetus-wrapper
@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o pipefail
+
+## @description Print a message to stderr
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
+function yetus_error
+{
+ echo "$*" 1>&2
+}
+
+## @description Given a filename or dir, return the absolute version of it
+## @audience public
+## @stability stable
+## @param directory
+## @replaceable no
+## @return 0 success
+## @return 1 failure
+## @return stdout abspath
+function yetus_abs
+{
+ declare obj=$1
+ declare dir
+ declare fn
+
+ if [[ ! -e ${obj} ]]; then
+ return 1
+ elif [[ -d ${obj} ]]; then
+ dir=${obj}
+ else
+ dir=$(dirname -- "${obj}")
+ fn=$(basename -- "${obj}")
+ fn="/${fn}"
+ fi
+
+ dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
+ if [[ $? = 0 ]]; then
+ echo "${dir}${fn}"
+ return 0
+ fi
+ return 1
+}
+
+
+WANTED="$1"
+shift
+ARGV=("$@")
+
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.1.0}
+BIN=$(yetus_abs "${BASH_SOURCE-$0}")
+BINDIR=$(dirname "${BIN}")
+
+###
+### if YETUS_HOME is set, then try to use it
+###
+if [[ -n "${YETUS_HOME}"
+ && -x "${YETUS_HOME}/bin/${WANTED}" ]]; then
+ exec "${YETUS_HOME}/bin/${WANTED}" "${ARGV[@]}"
+fi
+
+#
+# this directory is ignored by git and maven
+#
+HADOOP_PATCHPROCESS=${HADOOP_PATCHPROCESS:-"${BINDIR}/../../patchprocess"}
+
+if [[ ! -d "${HADOOP_PATCHPROCESS}" ]]; then
+ mkdir -p "${HADOOP_PATCHPROCESS}"
+fi
+
+mytmpdir=$(yetus_abs "${HADOOP_PATCHPROCESS}")
+if [[ $? != 0 ]]; then
+ yetus_error "yetus-dl: Unable to cwd to ${HADOOP_PATCHPROCESS}"
+ exit 1
+fi
+HADOOP_PATCHPROCESS=${mytmpdir}
+
+##
+## if we've already DL'd it, then short cut
+##
+if [[ -x "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
+ exec "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
+fi
+
+##
+## need to DL, etc
+##
+
+BASEURL="https://dist.apache.org/repos/dist/release/yetus/${HADOOP_YETUS_VERSION}/"
+TARBALL="yetus-${HADOOP_YETUS_VERSION}-bin.tar"
+
+GPGBIN=$(command -v gpg)
+CURLBIN=$(command -v curl)
+
+pushd "${HADOOP_PATCHPROCESS}" >/dev/null
+if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: Cannot pushd to ${HADOOP_PATCHPROCESS}"
+ exit 1
+fi
+
+if [[ -n "${CURLBIN}" ]]; then
+ "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz"
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz"
+ exit 1
+ fi
+else
+ yetus_error "ERROR: yetus-dl requires curl."
+ exit 1
+fi
+
+if [[ -n "${GPGBIN}" ]]; then
+ mkdir -p .gpg
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg"
+ exit 1
+ fi
+ chmod -R 700 .gpg
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: Unable to chmod ${HADOOP_PATCHPROCESS}/.gpg"
+ exit 1
+ fi
+ "${CURLBIN}" -s -L -o KEYS_YETUS https://dist.apache.org/repos/dist/release/yetus/KEYS
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS"
+ exit 1
+ fi
+ "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc"
+ exit 1
+ fi
+ "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --import "${HADOOP_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: gpg unable to import ${HADOOP_PATCHPROCESS}/KEYS_YETUS"
+ exit 1
+ fi
+ "${GPGBIN}" --homedir "${HADOOP_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HADOOP_PATCHPROCESS} failed"
+ exit 1
+ fi
+fi
+
+gunzip -c "${TARBALL}.gz" | tar xpf -
+if [[ $? != 0 ]]; then
+ yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HADOOP_PATCHPROCESS} to try again."
+ exit 1
+fi
+
+if [[ -x "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" ]]; then
+ exec "${HADOOP_PATCHPROCESS}/yetus-${HADOOP_YETUS_VERSION}/bin/${WANTED}" "${ARGV[@]}"
+fi
+
+##
+## give up
+##
+yetus_error "ERROR: ${WANTED} is not part of Apache Yetus ${HADOOP_YETUS_VERSION}"
+exit 1
[18/50] [abbrv] hadoop git commit: HADOOP-11828. Implement the
Hitchhiker erasure coding algorithm. Contributed by Jack Liu Quan.
Posted by ka...@apache.org.
HADOOP-11828. Implement the Hitchhiker erasure coding algorithm. Contributed by Jack Liu Quan.
Change-Id: If43475ccc2574df60949c947af562722db076251
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1bb31fb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1bb31fb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1bb31fb2
Branch: refs/heads/YARN-1011
Commit: 1bb31fb22e6f8e6df8e9ff4e94adf20308b4c743
Parents: 2ac39ca
Author: Zhe Zhang <zh...@apache.org>
Authored: Thu Jan 21 10:30:05 2016 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Thu Jan 21 10:30:05 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../coder/AbstractErasureDecoder.java | 2 +-
.../coder/AbstractHHErasureCodingStep.java | 49 +++
.../erasurecode/coder/HHXORErasureDecoder.java | 95 +++++
.../coder/HHXORErasureDecodingStep.java | 349 +++++++++++++++++++
.../erasurecode/coder/HHXORErasureEncoder.java | 92 +++++
.../coder/HHXORErasureEncodingStep.java | 146 ++++++++
.../io/erasurecode/coder/util/HHUtil.java | 216 ++++++++++++
.../erasurecode/coder/TestErasureCoderBase.java | 4 +-
.../coder/TestHHErasureCoderBase.java | 61 ++++
.../coder/TestHHXORErasureCoder.java | 120 +++++++
11 files changed, 1134 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 19ef66e..e5172d5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -262,6 +262,9 @@ Trunk (Unreleased)
HADOOP-11887. Introduce Intel ISA-L erasure coding library for native
erasure encoding support (Kai Zheng via Colin P. McCabe)
+ HADOOP-11828. Implement the Hitchhiker erasure coding algorithm.
+ (Jack Liuquan via zhz)
+
BUG FIXES
HADOOP-12617. SPNEGO authentication request to non-default realm gets
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
index abada3d..d976dd1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -56,7 +56,7 @@ public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
* We have all the data blocks and parity blocks as input blocks for
* recovering by default. It's codec specific
* @param blockGroup
- * @return
+ * @return input blocks
*/
protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
ECBlock[] inputBlocks = new ECBlock[getNumDataUnits() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java
new file mode 100644
index 0000000..e577c5d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractHHErasureCodingStep.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+
+/**
+ * Abstract class for Hitchhiker common facilities shared by
+ * {@link HHXORErasureEncodingStep}and {@link HHXORErasureDecodingStep}.
+ *
+ * It implements {@link AbstractErasureCodingStep}.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHHErasureCodingStep
+ extends AbstractErasureCodingStep {
+
+ private static final int SUB_PACKET_SIZE = 2;
+
+ /**
+ * Constructor given input blocks and output blocks.
+ *
+ * @param inputBlocks
+ * @param outputBlocks
+ */
+ public AbstractHHErasureCodingStep(ECBlock[] inputBlocks,
+ ECBlock[] outputBlocks) {
+ super(inputBlocks, outputBlocks);
+ }
+
+ protected int getSubPacketSize() {
+ return SUB_PACKET_SIZE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
new file mode 100644
index 0000000..ac4df16
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.*;
+
+/**
+ * Hitchhiker is a new erasure coding algorithm developed as a research project
+ * at UC Berkeley by Rashmi Vinayak.
+ * It has been shown to reduce network traffic and disk I/O by 25%-45% during
+ * data reconstruction while retaining the same storage capacity and failure
+ * tolerance capability of RS codes.
+ * The Hitchhiker algorithm is described in K.V.Rashmi, et al.,
+ * "A "Hitchhiker's" Guide to Fast and Efficient Data Reconstruction in
+ * Erasure-coded Data Centers", in ACM SIGCOMM 2014.
+ * This is Hitchhiker-XOR erasure decoder that decodes a block group.
+ */
+@InterfaceAudience.Private
+public class HHXORErasureDecoder extends AbstractErasureDecoder {
+ private RawErasureDecoder rsRawDecoder;
+ private RawErasureEncoder xorRawEncoder;
+
+ public HHXORErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public HHXORErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareDecodingStep(
+ final ECBlockGroup blockGroup) {
+
+ RawErasureDecoder rawDecoder;
+ RawErasureEncoder rawEncoder;
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+ ECBlock[] outputBlocks = getOutputBlocks(blockGroup);
+
+ rawDecoder = checkCreateRSRawDecoder();
+ rawEncoder = checkCreateXorRawEncoder();
+
+ return new HHXORErasureDecodingStep(inputBlocks,
+ getErasedIndexes(inputBlocks), outputBlocks, rawDecoder,
+ rawEncoder);
+ }
+
+ private RawErasureDecoder checkCreateRSRawDecoder() {
+ if (rsRawDecoder == null) {
+ rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+ }
+ return rsRawDecoder;
+ }
+
+ private RawErasureEncoder checkCreateXorRawEncoder() {
+ if (xorRawEncoder == null) {
+ xorRawEncoder = CodecUtil.createXORRawEncoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+ xorRawEncoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, false);
+ }
+ return xorRawEncoder;
+ }
+
+ @Override
+ public void release() {
+ if (rsRawDecoder != null) {
+ rsRawDecoder.release();
+ }
+ if (xorRawEncoder != null) {
+ xorRawEncoder.release();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
new file mode 100644
index 0000000..6c81836
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.coder.util.HHUtil;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Hitchhiker-XOR Erasure decoding step, a wrapper of all the necessary
+ * information to perform a decoding step involved in the whole process of
+ * decoding a block group.
+ */
+@InterfaceAudience.Private
+public class HHXORErasureDecodingStep extends AbstractHHErasureCodingStep {
+ private int pbIndex;
+ private int[] piggyBackIndex;
+ private int[] piggyBackFullIndex;
+ private int[] erasedIndexes;
+ private RawErasureDecoder rsRawDecoder;
+ private RawErasureEncoder xorRawEncoder;
+
+ /**
+ * The constructor with all the necessary info.
+ * @param inputBlocks
+ * @param erasedIndexes the indexes of erased blocks in inputBlocks array
+ * @param outputBlocks
+ * @param rawDecoder underlying RS decoder for hitchhiker decoding
+ * @param rawEncoder underlying XOR encoder for hitchhiker decoding
+ */
+ public HHXORErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
+ ECBlock[] outputBlocks, RawErasureDecoder rawDecoder,
+ RawErasureEncoder rawEncoder) {
+ super(inputBlocks, outputBlocks);
+ this.pbIndex = rawDecoder.getNumParityUnits() - 1;
+ this.erasedIndexes = erasedIndexes;
+ this.rsRawDecoder = rawDecoder;
+ this.xorRawEncoder = rawEncoder;
+
+ this.piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec(
+ rawDecoder.getNumDataUnits(), rawDecoder.getNumParityUnits());
+ this.piggyBackFullIndex = HHUtil.initPiggyBackFullIndexVec(
+ rawDecoder.getNumDataUnits(), piggyBackIndex);
+ }
+
+ @Override
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ if (erasedIndexes.length == 0) {
+ return;
+ }
+
+ ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputChunks);
+ ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputChunks);
+ performCoding(inputBuffers, outputBuffers);
+ }
+
+ private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ final int numDataUnits = rsRawDecoder.getNumDataUnits();
+ final int numParityUnits = rsRawDecoder.getNumParityUnits();
+ final int numTotalUnits = numDataUnits + numParityUnits;
+ final int subPacketSize = getSubPacketSize();
+
+ ByteBuffer fisrtValidInput = HHUtil.findFirstValidInput(inputs);
+ final int bufSize = fisrtValidInput.remaining();
+
+ if (inputs.length != numTotalUnits * getSubPacketSize()) {
+ throw new IllegalArgumentException("Invalid inputs length");
+ }
+
+ if (outputs.length != erasedIndexes.length * getSubPacketSize()) {
+ throw new IllegalArgumentException("Invalid outputs length");
+ }
+
+ // notes:inputs length = numDataUnits * subPacketizationSize
+ // first numDataUnits length is first sub-stripe,
+ // second numDataUnits length is second sub-stripe
+ ByteBuffer[][] newIn = new ByteBuffer[subPacketSize][numTotalUnits];
+ for (int i = 0; i < subPacketSize; ++i) {
+ for (int j = 0; j < numTotalUnits; ++j) {
+ newIn[i][j] = inputs[i * numTotalUnits + j];
+ }
+ }
+
+ ByteBuffer[][] newOut = new ByteBuffer[subPacketSize][erasedIndexes.length];
+ for (int i = 0; i < subPacketSize; ++i) {
+ for (int j = 0; j < erasedIndexes.length; ++j) {
+ newOut[i][j] = outputs[i * erasedIndexes.length + j];
+ }
+ }
+
+ if (erasedIndexes.length == 1 && erasedIndexes[0] < numDataUnits) {
+ // Only reconstruct one data unit missing
+ doDecodeSingle(newIn, newOut, erasedIndexes[0], bufSize,
+ fisrtValidInput.isDirect());
+ } else {
+ doDecodeMultiAndParity(newIn, newOut, erasedIndexes, bufSize);
+ }
+ }
+
+ private void doDecodeSingle(ByteBuffer[][] inputs, ByteBuffer[][] outputs,
+ int erasedLocationToFix, int bufSize,
+ boolean isDirect) {
+ final int numDataUnits = rsRawDecoder.getNumDataUnits();
+ final int numParityUnits = rsRawDecoder.getNumParityUnits();
+ final int subPacketSize = getSubPacketSize();
+
+ int[][] inputPositions = new int[subPacketSize][inputs[0].length];
+ for (int i = 0; i < subPacketSize; ++i) {
+ for (int j = 0; j < inputs[i].length; ++j) {
+ if (inputs[i][j] != null) {
+ inputPositions[i][j] = inputs[i][j].position();
+ }
+ }
+ }
+
+ ByteBuffer[] tempInputs = new ByteBuffer[numDataUnits + numParityUnits];
+ for (int i = 0; i < tempInputs.length; ++i) {
+ tempInputs[i] = inputs[1][i];
+ }
+
+ ByteBuffer[][] tmpOutputs = new ByteBuffer[subPacketSize][numParityUnits];
+ for (int i = 0; i < getSubPacketSize(); ++i) {
+ for (int j = 0; j < erasedIndexes.length; ++j) {
+ tmpOutputs[i][j] = outputs[i][j];
+ }
+
+ for (int m = erasedIndexes.length; m < numParityUnits; ++m) {
+ tmpOutputs[i][m] = HHUtil.allocateByteBuffer(isDirect, bufSize);
+ }
+ }
+
+ // First consider the second subPacket
+ int[] erasedLocation = new int[numParityUnits];
+ erasedLocation[0] = erasedLocationToFix;
+
+ // assign the erased locations based on the locations not read for
+ // second subPacket but from decoding
+ for (int i = 1; i < numParityUnits; i++) {
+ erasedLocation[i] = numDataUnits + i;
+ tempInputs[numDataUnits + i] = null;
+ }
+
+ rsRawDecoder.decode(tempInputs, erasedLocation, tmpOutputs[1]);
+
+ int piggyBackParityIndex = piggyBackFullIndex[erasedLocationToFix];
+ ByteBuffer piggyBack = HHUtil.getPiggyBackForDecode(inputs, tmpOutputs,
+ piggyBackParityIndex, numDataUnits, numParityUnits, pbIndex);
+
+ // Second consider the first subPacket.
+ // get the value of the piggyback associated with the erased location
+ if (isDirect) {
+ // decode the erased value in the first subPacket by using the piggyback
+ int idxToWrite = 0;
+ doDecodeByPiggyBack(inputs[0], tmpOutputs[0][idxToWrite], piggyBack,
+ erasedLocationToFix);
+ } else {
+ ByteBuffer buffer;
+ byte[][][] newInputs = new byte[getSubPacketSize()][inputs[0].length][];
+ int[][] inputOffsets = new int[getSubPacketSize()][inputs[0].length];
+ byte[][][] newOutputs = new byte[getSubPacketSize()][numParityUnits][];
+ int[][] outOffsets = new int[getSubPacketSize()][numParityUnits];
+
+ for (int i = 0; i < getSubPacketSize(); ++i) {
+ for (int j = 0; j < inputs[0].length; ++j) {
+ buffer = inputs[i][j];
+ if (buffer != null) {
+ inputOffsets[i][j] = buffer.arrayOffset() + buffer.position();
+ newInputs[i][j] = buffer.array();
+ }
+ }
+ }
+
+ for (int i = 0; i < getSubPacketSize(); ++i) {
+ for (int j = 0; j < numParityUnits; ++j) {
+ buffer = tmpOutputs[i][j];
+ if (buffer != null) {
+ outOffsets[i][j] = buffer.arrayOffset() + buffer.position();
+ newOutputs[i][j] = buffer.array();
+ }
+ }
+ }
+
+ byte[] newPiggyBack = piggyBack.array();
+
+ // decode the erased value in the first subPacket by using the piggyback
+ int idxToWrite = 0;
+ doDecodeByPiggyBack(newInputs[0], inputOffsets[0],
+ newOutputs[0][idxToWrite], outOffsets[0][idxToWrite],
+ newPiggyBack, erasedLocationToFix, bufSize);
+ }
+
+ for (int i = 0; i < subPacketSize; ++i) {
+ for (int j = 0; j < inputs[i].length; ++j) {
+ if (inputs[i][j] != null) {
+ inputs[i][j].position(inputPositions[i][j] + bufSize);
+ }
+ }
+ }
+ }
+
+ private void doDecodeByPiggyBack(ByteBuffer[] inputs,
+ ByteBuffer outputs,
+ ByteBuffer piggyBack,
+ int erasedLocationToFix) {
+ final int thisPiggyBackSetIdx = piggyBackFullIndex[erasedLocationToFix];
+ final int startIndex = piggyBackIndex[thisPiggyBackSetIdx - 1];
+ final int endIndex = piggyBackIndex[thisPiggyBackSetIdx];
+
+ // recover first sub-stripe data by XOR piggyback
+ int bufSize = piggyBack.remaining();
+ for (int i = piggyBack.position();
+ i < piggyBack.position() + bufSize; i++) {
+ for (int j = startIndex; j < endIndex; j++) {
+ if (inputs[j] != null) {
+ piggyBack.put(i, (byte)
+ (piggyBack.get(i) ^ inputs[j].get(inputs[j].position() + i)));
+ }
+ }
+ outputs.put(outputs.position() + i, piggyBack.get(i));
+ }
+ }
+
+ private void doDecodeByPiggyBack(byte[][] inputs, int[] inputOffsets,
+ byte[] outputs, int outOffset,
+ byte[] piggyBack, int erasedLocationToFix,
+ int bufSize) {
+ final int thisPiggyBackSetIdx = piggyBackFullIndex[erasedLocationToFix];
+ final int startIndex = piggyBackIndex[thisPiggyBackSetIdx - 1];
+ final int endIndex = piggyBackIndex[thisPiggyBackSetIdx];
+
+ // recover first sub-stripe data by XOR piggyback
+ for (int i = 0; i < bufSize; i++) {
+ for (int j = startIndex; j < endIndex; j++) {
+ if (inputs[j] != null) {
+ piggyBack[i] = (byte) (piggyBack[i] ^ inputs[j][i + inputOffsets[j]]);
+ }
+ }
+ outputs[i + outOffset] = piggyBack[i];
+ }
+ }
+
+ private void doDecodeMultiAndParity(ByteBuffer[][] inputs,
+ ByteBuffer[][] outputs,
+ int[] erasedLocationToFix, int bufSize) {
+ final int numDataUnits = rsRawDecoder.getNumDataUnits();
+ final int numParityUnits = rsRawDecoder.getNumParityUnits();
+ final int numTotalUnits = numDataUnits + numParityUnits;
+ int[] parityToFixFlag = new int[numTotalUnits];
+
+ for (int i = 0; i < erasedLocationToFix.length; ++i) {
+ if (erasedLocationToFix[i] >= numDataUnits) {
+ parityToFixFlag[erasedLocationToFix[i]] = 1;
+ }
+ }
+
+ int[] inputPositions = new int[inputs[0].length];
+ for (int i = 0; i < inputPositions.length; i++) {
+ if (inputs[0][i] != null) {
+ inputPositions[i] = inputs[0][i].position();
+ }
+ }
+
+ // decoded first sub-stripe
+ rsRawDecoder.decode(inputs[0], erasedLocationToFix, outputs[0]);
+
+ for (int i = 0; i < inputs[0].length; i++) {
+ if (inputs[0][i] != null) {
+ // dataLen bytes consumed
+ inputs[0][i].position(inputPositions[i]);
+ }
+ }
+
+ ByteBuffer[] tempInput = new ByteBuffer[numDataUnits];
+ for (int i = 0; i < numDataUnits; ++i) {
+ tempInput[i] = inputs[0][i];
+//
+// if (!isDirect && tempInput[i] != null) {
+// tempInput[i].position(tempInput[i].position() - bufSize);
+// }
+ }
+
+ for (int i = 0; i < erasedLocationToFix.length; ++i) {
+ if (erasedLocationToFix[i] < numDataUnits) {
+ tempInput[erasedLocationToFix[i]] = outputs[0][i];
+ }
+ }
+
+ ByteBuffer[] piggyBack = HHUtil.getPiggyBacksFromInput(tempInput,
+ piggyBackIndex, numParityUnits, 0, xorRawEncoder);
+
+ for (int j = numDataUnits + 1; j < numTotalUnits; ++j) {
+ if (parityToFixFlag[j] == 0 && inputs[1][j] != null) {
+ // f(b) + f(a1,a2,a3....)
+ for (int k = inputs[1][j].position(),
+ m = piggyBack[j - numDataUnits - 1].position();
+ k < inputs[1][j].limit(); ++k, ++m) {
+ inputs[1][j].put(k, (byte)
+ (inputs[1][j].get(k) ^
+ piggyBack[j - numDataUnits - 1].get(m)));
+ }
+ }
+ }
+
+ // decoded second sub-stripe
+ rsRawDecoder.decode(inputs[1], erasedLocationToFix, outputs[1]);
+
+ // parity index = 0, the data have no piggyBack
+ for (int j = 0; j < erasedLocationToFix.length; ++j) {
+ if (erasedLocationToFix[j] < numTotalUnits
+ && erasedLocationToFix[j] > numDataUnits) {
+ int parityIndex = erasedLocationToFix[j] - numDataUnits - 1;
+ for (int k = outputs[1][j].position(),
+ m = piggyBack[parityIndex].position();
+ k < outputs[1][j].limit(); ++k, ++m) {
+ outputs[1][j].put(k, (byte)
+ (outputs[1][j].get(k) ^ piggyBack[parityIndex].get(m)));
+ }
+ }
+ }
+
+ for (int i = 0; i < inputs[0].length; i++) {
+ if (inputs[0][i] != null) {
+ // dataLen bytes consumed
+ inputs[0][i].position(inputPositions[i] + bufSize);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
new file mode 100644
index 0000000..f30572f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.CoderOption;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Hitchhiker is a new erasure coding algorithm developed as a research project
+ * at UC Berkeley by Rashmi Vinayak.
+ * It has been shown to reduce network traffic and disk I/O by 25%-45% during
+ * data reconstruction while retaining the same storage capacity and failure
+ * tolerance capability of RS codes.
+ * The Hitchhiker algorithm is described in K.V.Rashmi, et al.,
+ * "A "Hitchhiker's" Guide to Fast and Efficient Data Reconstruction in
+ * Erasure-coded Data Centers", in ACM SIGCOMM 2014.
+ * This is Hitchhiker-XOR erasure encoder that encodes a block group.
+ */
+@InterfaceAudience.Private
+public class HHXORErasureEncoder extends AbstractErasureEncoder {
+ private RawErasureEncoder rsRawEncoder;
+ private RawErasureEncoder xorRawEncoder;
+
+ public HHXORErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public HHXORErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareEncodingStep(
+ final ECBlockGroup blockGroup) {
+
+ RawErasureEncoder rsRawEncoderTmp = checkCreateRSRawEncoder();
+ RawErasureEncoder xorRawEncoderTmp = checkCreateXorRawEncoder();
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new HHXORErasureEncodingStep(inputBlocks,
+ getOutputBlocks(blockGroup), rsRawEncoderTmp, xorRawEncoderTmp);
+ }
+
+ private RawErasureEncoder checkCreateRSRawEncoder() {
+ if (rsRawEncoder == null) {
+ rsRawEncoder = CodecUtil.createRSRawEncoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+ }
+ return rsRawEncoder;
+ }
+
+ private RawErasureEncoder checkCreateXorRawEncoder() {
+ if (xorRawEncoder == null) {
+ xorRawEncoder = CodecUtil.createXORRawEncoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+ xorRawEncoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, false);
+ }
+ return xorRawEncoder;
+ }
+
+ @Override
+ public void release() {
+ if (rsRawEncoder != null) {
+ rsRawEncoder.release();
+ }
+ if (xorRawEncoder != null) {
+ xorRawEncoder.release();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
new file mode 100644
index 0000000..f83ee26
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.coder.util.HHUtil;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Hitchhiker-XOR Erasure encoding step, a wrapper of all the necessary
+ * information to perform an encoding step involved in the whole process of
+ * encoding a block group.
+ */
+@InterfaceAudience.Private
+public class HHXORErasureEncodingStep extends AbstractHHErasureCodingStep {
+ private int[] piggyBackIndex;
+ private RawErasureEncoder rsRawEncoder;
+ private RawErasureEncoder xorRawEncoder;
+
+ /**
+ * The constructor with all the necessary info.
+ *
+ * @param inputBlocks
+ * @param outputBlocks
+ * @param rsRawEncoder underlying RS encoder for hitchhiker encoding
+ * @param xorRawEncoder underlying XOR encoder for hitchhiker encoding
+ */
+ public HHXORErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
+ RawErasureEncoder rsRawEncoder,
+ RawErasureEncoder xorRawEncoder) {
+ super(inputBlocks, outputBlocks);
+
+ this.rsRawEncoder = rsRawEncoder;
+ this.xorRawEncoder = xorRawEncoder;
+ piggyBackIndex = HHUtil.initPiggyBackIndexWithoutPBVec(
+ rsRawEncoder.getNumDataUnits(), rsRawEncoder.getNumParityUnits());
+ }
+
+ @Override
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputChunks);
+ ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputChunks);
+ performCoding(inputBuffers, outputBuffers);
+ }
+
+ private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ final int numDataUnits = this.rsRawEncoder.getNumDataUnits();
+ final int numParityUnits = this.rsRawEncoder.getNumParityUnits();
+ final int subSPacketSize = getSubPacketSize();
+
+ // inputs length = numDataUnits * subPacketSize
+ if (inputs.length != numDataUnits * subSPacketSize) {
+ throw new IllegalArgumentException("Invalid inputs length");
+ }
+
+ if (outputs.length != numParityUnits * subSPacketSize) {
+ throw new IllegalArgumentException("Invalid outputs length");
+ }
+
+ // first numDataUnits length is first sub-stripe,
+ // second numDataUnits length is second sub-stripe
+ ByteBuffer[][] hhInputs = new ByteBuffer[subSPacketSize][numDataUnits];
+ for (int i = 0; i < subSPacketSize; ++i) {
+ for (int j = 0; j < numDataUnits; ++j) {
+ hhInputs[i][j] = inputs[i * numDataUnits + j];
+ }
+ }
+
+ ByteBuffer[][] hhOutputs = new ByteBuffer[subSPacketSize][numParityUnits];
+ for (int i = 0; i < subSPacketSize; ++i) {
+ for (int j = 0; j < numParityUnits; ++j) {
+ hhOutputs[i][j] = outputs[i * numParityUnits + j];
+ }
+ }
+
+ doEncode(hhInputs, hhOutputs);
+ }
+
+ private void doEncode(ByteBuffer[][] inputs, ByteBuffer[][] outputs) {
+ final int numParityUnits = this.rsRawEncoder.getNumParityUnits();
+
+ // calc piggyBacks using first sub-packet
+ ByteBuffer[] piggyBacks = HHUtil.getPiggyBacksFromInput(inputs[0],
+ piggyBackIndex, numParityUnits, 0, xorRawEncoder);
+
+ // Step1: RS encode each byte-stripe of sub-packets
+ for (int i = 0; i < getSubPacketSize(); ++i) {
+ rsRawEncoder.encode(inputs[i], outputs[i]);
+ }
+
+ // Step2: Adding piggybacks to the parities
+ // Only second sub-packet is added with a piggyback.
+ encodeWithPiggyBacks(piggyBacks, outputs, numParityUnits,
+ inputs[0][0].isDirect());
+ }
+
+ private void encodeWithPiggyBacks(ByteBuffer[] piggyBacks,
+ ByteBuffer[][] outputs,
+ int numParityUnits,
+ boolean bIsDirect) {
+ if (!bIsDirect) {
+ for (int i = 0; i < numParityUnits - 1; i++) {
+ int parityIndex = i + 1;
+ int bufSize = piggyBacks[i].remaining();
+ byte[] newOut = outputs[1][parityIndex].array();
+ int offset = outputs[1][parityIndex].arrayOffset()
+ + outputs[1][parityIndex].position();
+
+ for (int k = offset, j = 0; j < bufSize; k++, j++) {
+ newOut[k] = (byte) (newOut[k] ^ piggyBacks[i].get(j));
+ }
+ }
+ return;
+ }
+
+ for (int i = 0; i < numParityUnits - 1; i++) {
+ int parityIndex = i + 1;
+ for (int k = piggyBacks[i].position(),
+ m = outputs[1][parityIndex].position();
+ k < piggyBacks[i].limit(); k++, m++) {
+ outputs[1][parityIndex].put(m,
+ (byte) (outputs[1][parityIndex].get(m) ^ piggyBacks[i].get(k)));
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
new file mode 100644
index 0000000..cfb567e3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder.util;
+
+import java.nio.ByteBuffer;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
+
+/**
+ * Some utilities for Hitchhiker coding.
+ */
+@InterfaceAudience.Private
+public final class HHUtil {
+ private HHUtil() {
+ // No called
+ }
+
+ public static int[] initPiggyBackIndexWithoutPBVec(int numDataUnits,
+ int numParityUnits) {
+ final int piggyBackSize = numDataUnits / (numParityUnits - 1);
+ int[] piggyBackIndex = new int[numParityUnits];
+
+ for (int i = 0; i < numDataUnits; ++i) {
+ if ((i % piggyBackSize) == 0) {
+ piggyBackIndex[i / piggyBackSize] = i;
+ }
+ }
+
+ piggyBackIndex[numParityUnits - 1] = numDataUnits;
+ return piggyBackIndex;
+ }
+
+ public static int[] initPiggyBackFullIndexVec(int numDataUnits,
+ int[] piggyBackIndex) {
+ int[] piggyBackFullIndex = new int[numDataUnits];
+
+ for (int i = 1; i < piggyBackIndex.length; ++i) {
+ for (int j = piggyBackIndex[i - 1]; j < piggyBackIndex[i]; ++j) {
+ piggyBackFullIndex[j] = i;
+ }
+ }
+
+ return piggyBackFullIndex;
+ }
+
+ public static ByteBuffer[] getPiggyBacksFromInput(ByteBuffer[] inputs,
+ int[] piggyBackIndex,
+ int numParityUnits,
+ int pgIndex,
+ RawErasureEncoder encoder) {
+ ByteBuffer[] emptyInput = new ByteBuffer[inputs.length];
+ ByteBuffer[] tempInput = new ByteBuffer[inputs.length];
+ int[] inputPositions = new int[inputs.length];
+
+ for (int m = 0; m < inputs.length; ++m) {
+ if (inputs[m] != null) {
+ emptyInput[m] = allocateByteBuffer(inputs[m].isDirect(),
+ inputs[m].remaining());
+ }
+ }
+
+ ByteBuffer[] tempOutput = new ByteBuffer[numParityUnits];
+ for (int m = 0; m < numParityUnits; ++m) {
+ tempOutput[m] = allocateByteBuffer(inputs[m].isDirect(),
+ inputs[0].remaining());
+ }
+
+ ByteBuffer[] piggyBacks = new ByteBuffer[numParityUnits - 1];
+ assert (piggyBackIndex.length >= numParityUnits);
+
+ // using underlying RS code to create piggybacks
+ for (int i = 0; i < numParityUnits - 1; ++i) {
+ for (int k = piggyBackIndex[i]; k < piggyBackIndex[i + 1]; ++k) {
+ tempInput[k] = inputs[k];
+ inputPositions[k] = inputs[k].position();
+ }
+ for (int n = 0; n < emptyInput.length; ++n) {
+ if (tempInput[n] == null) {
+ tempInput[n] = emptyInput[n];
+ inputPositions[n] = emptyInput[n].position();
+ }
+ }
+
+ encoder.encode(tempInput, tempOutput);
+
+ piggyBacks[i] = cloneBufferData(tempOutput[pgIndex]);
+
+ for (int j = 0; j < tempInput.length; j++) {
+ if (tempInput[j] != null) {
+ tempInput[j].position(inputPositions[j]);
+ tempInput[j] = null;
+ }
+ }
+
+ for (int j = 0; j < tempOutput.length; j++) {
+ tempOutput[j].clear();
+ }
+ }
+
+ return piggyBacks;
+ }
+
+ private static ByteBuffer cloneBufferData(ByteBuffer srcBuffer) {
+ ByteBuffer destBuffer;
+ byte[] bytesArr = new byte[srcBuffer.remaining()];
+
+ srcBuffer.mark();
+ srcBuffer.get(bytesArr);
+ srcBuffer.reset();
+
+ if (!srcBuffer.isDirect()) {
+ destBuffer = ByteBuffer.wrap(bytesArr);
+ } else {
+ destBuffer = ByteBuffer.allocateDirect(srcBuffer.remaining());
+ destBuffer.put(bytesArr);
+ destBuffer.flip();
+ }
+
+ return destBuffer;
+ }
+
+ public static ByteBuffer allocateByteBuffer(boolean useDirectBuffer,
+ int bufSize) {
+ if (useDirectBuffer) {
+ return ByteBuffer.allocateDirect(bufSize);
+ } else {
+ return ByteBuffer.allocate(bufSize);
+ }
+ }
+
+ public static ByteBuffer getPiggyBackForDecode(ByteBuffer[][] inputs,
+ ByteBuffer[][] outputs,
+ int pbParityIndex,
+ int numDataUnits,
+ int numParityUnits,
+ int pbIndex) {
+ ByteBuffer fisrtValidInput = HHUtil.findFirstValidInput(inputs[0]);
+ int bufSize = fisrtValidInput.remaining();
+
+ ByteBuffer piggybacks = allocateByteBuffer(fisrtValidInput.isDirect(),
+ bufSize);
+
+ // Use piggyBackParityIndex to figure out which parity location has the
+ // associated piggyBack
+ // Obtain the piggyback by subtracting the decoded (second sub-packet
+ // only ) parity value from the actually read parity value
+ if (pbParityIndex < numParityUnits) {
+ // not the last piggybackSet
+ int inputIdx = numDataUnits + pbParityIndex;
+ int inputPos = inputs[1][inputIdx].position();
+ int outputPos = outputs[1][pbParityIndex].position();
+
+ for (int m = 0, k = inputPos, n = outputPos; m < bufSize; k++, m++, n++) {
+ int valueWithPb = 0xFF & inputs[1][inputIdx].get(k);
+ int valueWithoutPb = 0xFF & outputs[1][pbParityIndex].get(n);
+ piggybacks.put(m, (byte) RSUtil.GF.add(valueWithPb, valueWithoutPb));
+ }
+ } else {
+ // last piggybackSet
+ int sum = 0;
+ for (int k = 0; k < bufSize; k++) {
+ sum = 0;
+ for (int i = 1; i < numParityUnits; i++) {
+ int inIdx = numDataUnits + i;
+ int inPos = inputs[1][numDataUnits + i].position();
+ int outPos = outputs[1][i].position();
+
+ sum = RSUtil.GF.add(sum, (0xFF & inputs[1][inIdx].get(inPos + k)));
+ sum = RSUtil.GF.add(sum, (0xFF & outputs[1][i].get(outPos + k)));
+ }
+
+ sum = RSUtil.GF.add(sum,
+ (0xFF & inputs[0][numDataUnits + pbIndex].get(
+ inputs[0][numDataUnits + pbIndex].position() + k)));
+
+ piggybacks.put(k, (byte) sum);
+ }
+
+ }
+
+ return piggybacks;
+ }
+
+ /**
+ * Find the valid input from all the inputs.
+ * @param inputs input buffers to look for valid input
+ * @return the first valid input
+ */
+ public static <T> T findFirstValidInput(T[] inputs) {
+ for (T input : inputs) {
+ if (input != null) {
+ return input;
+ }
+ }
+
+ throw new HadoopIllegalArgumentException(
+ "Invalid inputs are found, all being null");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
index 0584977..261be57 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -41,7 +41,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* at all for simple.
*/
protected static class TestBlock extends ECBlock {
- private ECChunk[] chunks;
+ protected ECChunk[] chunks;
// For simple, just assume the block have the chunks already ready.
// In practice we need to read/write chunks from/to the block via file IO.
@@ -101,7 +101,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* This is typically how a coding step should be performed.
* @param codingStep
*/
- private void performCodingStep(ErasureCodingStep codingStep) {
+ protected void performCodingStep(ErasureCodingStep codingStep) {
// Pretend that we're opening these input blocks and output blocks.
ECBlock[] inputBlocks = codingStep.getInputBlocks();
ECBlock[] outputBlocks = codingStep.getOutputBlocks();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
new file mode 100644
index 0000000..5471153
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+
+/**
+ * Erasure coder test base with utilities for hitchhiker.
+ */
+public abstract class TestHHErasureCoderBase extends TestErasureCoderBase{
+ protected int subPacketSize = 2;
+
+ @Override
+ protected void performCodingStep(ErasureCodingStep codingStep) {
+ // Pretend that we're opening these input blocks and output blocks.
+ ECBlock[] inputBlocks = codingStep.getInputBlocks();
+ ECBlock[] outputBlocks = codingStep.getOutputBlocks();
+ // We allocate input and output chunks accordingly.
+ ECChunk[] inputChunks = new ECChunk[inputBlocks.length * subPacketSize];
+ ECChunk[] outputChunks = new ECChunk[outputBlocks.length * subPacketSize];
+
+ for (int i = 0; i < numChunksInBlock; i += subPacketSize) {
+ // Pretend that we're reading input chunks from input blocks.
+ for (int k = 0; k < subPacketSize; ++k) {
+ for (int j = 0; j < inputBlocks.length; ++j) {
+ inputChunks[k * inputBlocks.length + j] = ((TestBlock)
+ inputBlocks[j]).chunks[i + k];
+ }
+
+ // Pretend that we allocate and will write output results to the blocks.
+ for (int j = 0; j < outputBlocks.length; ++j) {
+ outputChunks[k * outputBlocks.length + j] = allocateOutputChunk();
+ ((TestBlock) outputBlocks[j]).chunks[i + k] =
+ outputChunks[k * outputBlocks.length + j];
+ }
+ }
+
+ // Given the input chunks and output chunk buffers, just call it !
+ codingStep.performCoding(inputChunks, outputChunks);
+ }
+
+ codingStep.finish();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bb31fb2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
new file mode 100644
index 0000000..ad346e0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
+
+ @Before
+ public void setup() {
+ this.encoderClass = HHXORErasureEncoder.class;
+ this.decoderClass = HHXORErasureDecoder.class;
+ this.numChunksInBlock = 10;
+ this.subPacketSize = 2;
+ }
+
+ @Test
+ public void testCodingNoDirectBuffer_10x4_erasing_d0() {
+ prepare(null, 10, 4, new int[]{0}, new int[0]);
+ /**
+ * Doing twice to test if the coders can be repeatedly reused. This matters
+ * as the underlying coding buffers are shared, which may have bugs.
+ */
+ testCoding(false);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBufferWithConf_10x4_erasing_d0() {
+ /**
+ * This tests if the configuration items work or not.
+ */
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ RSRawErasureCoderFactory.class.getCanonicalName());
+ prepare(conf, 10, 4, new int[]{0}, new int[0]);
+
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_p1() {
+ prepare(null, 10, 4, new int[]{}, new int[]{1});
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_d4() {
+ prepare(null, 10, 4, new int[] {4}, new int[] {});
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_d0_p0() {
+ prepare(null, 10, 4, new int[] {0}, new int[] {0});
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingBothBuffers_10x4_erasing_d0_p0() {
+ prepare(null, 10, 4, new int[] {0}, new int[] {0});
+
+ /**
+ * Doing in mixed buffer usage model to test if the coders can be repeatedly
+ * reused with different buffer usage model. This matters as the underlying
+ * coding buffers are shared, which may have bugs.
+ */
+ testCoding(true);
+ testCoding(false);
+ testCoding(true);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
+ prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_d0_d1_p0_p1() {
+ prepare(null, 10, 4, new int[] {0, 1}, new int[] {0, 1});
+ testCoding(true);
+ }
+
+// @Test
+// public void testCodingNoDirectBuffer_3x3_erasing_d0_p0() {
+// prepare(null, 3, 3, new int[] {0}, new int[] {0});
+// testCoding(false);
+// }
+
+ @Test
+ public void testCodingDirectBuffer_6x3_erasing_d0_p0() {
+ prepare(null, 6, 3, new int[] {0}, new int[] {0});
+ testCoding(true);
+ }
+}
[03/50] [abbrv] hadoop git commit: HADOOP-12696. Add tests for
S3FileSystem Contract. Contributed by Matt Paduano
Posted by ka...@apache.org.
HADOOP-12696. Add tests for S3FileSystem Contract. Contributed by Matt Paduano
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1acc509b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1acc509b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1acc509b
Branch: refs/heads/YARN-1011
Commit: 1acc509b45d58c0eb7e83ea1ba13169410be0dbe
Parents: 89d1fd5
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Tue Jan 19 13:57:08 2016 -0800
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Tue Jan 19 13:57:08 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../src/site/markdown/filesystem/testing.md | 4 +-
.../fs/contract/AbstractContractSeekTest.java | 8 +-
.../hadoop/fs/contract/ContractOptions.java | 6 ++
.../org/apache/hadoop/fs/s3/S3FileSystem.java | 58 +++++++----
.../org/apache/hadoop/fs/s3/S3InputStream.java | 17 +--
.../hadoop/fs/contract/s3/S3Contract.java | 40 +++++++
.../fs/contract/s3/TestS3ContractCreate.java | 32 ++++++
.../fs/contract/s3/TestS3ContractDelete.java | 31 ++++++
.../fs/contract/s3/TestS3ContractMkdir.java | 32 ++++++
.../fs/contract/s3/TestS3ContractOpen.java | 32 ++++++
.../fs/contract/s3/TestS3ContractRename.java | 32 ++++++
.../fs/contract/s3/TestS3ContractRootDir.java | 34 ++++++
.../fs/contract/s3/TestS3ContractSeek.java | 32 ++++++
.../src/test/resources/contract/s3.xml | 104 +++++++++++++++++++
15 files changed, 430 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bd6550c..a5dcf6a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1078,6 +1078,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12604. Exception may be swallowed in KMSClientProvider.
(Yongjun Zhang)
+ HADOOP-12696. Add tests for S3Filesystem Contract (Matt Paduano via raviprak)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
index 444fb60..99561cd 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
@@ -190,7 +190,7 @@ tests against remote FileSystems that require login details require usernames/ID
All these details MUST be required to be placed in the file `src/test/resources/contract-test-options.xml`, and your SCM tools configured to never commit this file to subversion, git or
equivalent. Furthermore, the build MUST be configured to never bundle this file in any `-test` artifacts generated. The Hadoop build does this, excluding `src/test/**/*.xml` from the JAR files.
-
+In addition, `src/test/resources/auth-keys.xml` will need to be created. It can be a copy of `contract-test-options.xml`.
The `AbstractFSContract` class automatically loads this resource file if present; specific keys for specific test cases can be added.
As an example, here are what S3N test keys look like:
@@ -214,7 +214,7 @@ As an example, here are what S3N test keys look like:
The `AbstractBondedFSContract` automatically skips a test suite if the FileSystem URL is not defined in the property `fs.contract.test.fs.%s`, where `%s` matches the schema name of the FileSystem.
-
+When running the tests `maven.test.skip` will need to be turned off since it is true by default on these tests. This can be done with a command like `mvn test -Ptests-on`.
### Important: passing the tests does not guarantee compatibility
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
index 4a0560e..8f56510 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
@@ -116,15 +116,13 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas
*/
@Test
public void testSeekReadClosedFile() throws Throwable {
- boolean supportsSeekOnClosedFiles = isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE);
-
instream = getFileSystem().open(smallSeekFile);
getLog().debug(
"Stream is of type " + instream.getClass().getCanonicalName());
instream.close();
try {
instream.seek(0);
- if (!supportsSeekOnClosedFiles) {
+ if (!isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE)) {
fail("seek succeeded on a closed stream");
}
} catch (IOException e) {
@@ -132,7 +130,9 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas
}
try {
int data = instream.available();
- fail("read() succeeded on a closed stream, got " + data);
+ if (!isSupported(SUPPORTS_AVAILABLE_ON_CLOSED_FILE)) {
+ fail("available() succeeded on a closed stream, got " + data);
+ }
} catch (IOException e) {
//expected a closed file
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
index 56caced..d8c2592 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
@@ -148,6 +148,12 @@ public interface ContractOptions {
String SUPPORTS_SEEK_ON_CLOSED_FILE = "supports-seek-on-closed-file";
/**
+ * Is available() on a closed InputStream supported?
+ * @{value}
+ */
+ String SUPPORTS_AVAILABLE_ON_CLOSED_FILE = "supports-available-on-closed-file";
+
+ /**
* Flag to indicate that this FS expects to throw the strictest
* exceptions it can, not generic IOEs, which, if returned,
* must be rejected.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
index 8bdfe9a..e5147a3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
@@ -62,7 +63,7 @@ public class S3FileSystem extends FileSystem {
public S3FileSystem() {
// set store in initialize()
}
-
+
public S3FileSystem(FileSystemStore store) {
this.store = store;
}
@@ -90,14 +91,14 @@ public class S3FileSystem extends FileSystem {
}
store.initialize(uri, conf);
setConf(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
+ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.workingDir =
new Path("/user", System.getProperty("user.name")).makeQualified(this);
- }
+ }
private static FileSystemStore createDefaultStore(Configuration conf) {
FileSystemStore store = new Jets3tFileSystemStore();
-
+
RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
conf.getInt("fs.s3.maxRetries", 4),
conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
@@ -105,13 +106,13 @@ public class S3FileSystem extends FileSystem {
new HashMap<Class<? extends Exception>, RetryPolicy>();
exceptionToPolicyMap.put(IOException.class, basePolicy);
exceptionToPolicyMap.put(S3Exception.class, basePolicy);
-
+
RetryPolicy methodPolicy = RetryPolicies.retryByException(
RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
methodNameToPolicyMap.put("storeBlock", methodPolicy);
methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
-
+
return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
store, methodNameToPolicyMap);
}
@@ -144,21 +145,29 @@ public class S3FileSystem extends FileSystem {
paths.add(0, absolutePath);
absolutePath = absolutePath.getParent();
} while (absolutePath != null);
-
+
boolean result = true;
- for (Path p : paths) {
- result &= mkdir(p);
+ for (int i = 0; i < paths.size(); i++) {
+ Path p = paths.get(i);
+ try {
+ result &= mkdir(p);
+ } catch(FileAlreadyExistsException e) {
+ if (i + 1 < paths.size()) {
+ throw new ParentNotDirectoryException(e.getMessage());
+ }
+ throw e;
+ }
}
return result;
}
-
+
private boolean mkdir(Path path) throws IOException {
Path absolutePath = makeAbsolute(path);
INode inode = store.retrieveINode(absolutePath);
if (inode == null) {
store.storeINode(absolutePath, INode.DIRECTORY_INODE);
} else if (inode.isFile()) {
- throw new IOException(String.format(
+ throw new FileAlreadyExistsException(String.format(
"Can't make directory for path %s since it is a file.",
absolutePath));
}
@@ -176,11 +185,12 @@ public class S3FileSystem extends FileSystem {
private INode checkFile(Path path) throws IOException {
INode inode = store.retrieveINode(makeAbsolute(path));
+ String message = String.format("No such file: '%s'", path.toString());
if (inode == null) {
- throw new IOException("No such file.");
+ throw new FileNotFoundException(message + " does not exist");
}
if (inode.isDirectory()) {
- throw new IOException("Path " + path + " is a directory.");
+ throw new FileNotFoundException(message + " is a directory");
}
return inode;
}
@@ -222,10 +232,14 @@ public class S3FileSystem extends FileSystem {
INode inode = store.retrieveINode(makeAbsolute(file));
if (inode != null) {
- if (overwrite) {
+ if (overwrite && !inode.isDirectory()) {
delete(file, true);
} else {
- throw new FileAlreadyExistsException("File already exists: " + file);
+ String message = String.format("File already exists: '%s'", file);
+ if (inode.isDirectory()) {
+ message = message + " is a directory";
+ }
+ throw new FileAlreadyExistsException(message);
}
} else {
Path parent = file.getParent();
@@ -233,7 +247,7 @@ public class S3FileSystem extends FileSystem {
if (!mkdirs(parent)) {
throw new IOException("Mkdirs failed to create " + parent.toString());
}
- }
+ }
}
return new FSDataOutputStream
(new S3OutputStream(getConf(), store, makeAbsolute(file),
@@ -259,7 +273,7 @@ public class S3FileSystem extends FileSystem {
if (debugEnabled) {
LOG.debug(debugPreamble + "returning false as src does not exist");
}
- return false;
+ return false;
}
Path absoluteDst = makeAbsolute(dst);
@@ -404,7 +418,7 @@ public class S3FileSystem extends FileSystem {
store.deleteBlock(block);
}
} else {
- FileStatus[] contents = null;
+ FileStatus[] contents = null;
try {
contents = listStatus(absolutePath);
} catch(FileNotFoundException fnfe) {
@@ -412,7 +426,7 @@ public class S3FileSystem extends FileSystem {
}
if ((contents.length !=0) && (!recursive)) {
- throw new IOException("Directory " + path.toString()
+ throw new IOException("Directory " + path.toString()
+ " is not empty.");
}
for (FileStatus p:contents) {
@@ -424,9 +438,9 @@ public class S3FileSystem extends FileSystem {
}
return true;
}
-
+
/**
- * FileStatus for S3 file systems.
+ * FileStatus for S3 file systems.
*/
@Override
public FileStatus getFileStatus(Path f) throws IOException {
@@ -436,7 +450,7 @@ public class S3FileSystem extends FileSystem {
}
return new S3FileStatus(f.makeQualified(this), inode);
}
-
+
@Override
public long getDefaultBlockSize() {
return getConf().getLong("fs.s3.block.size", 64 * 1024 * 1024);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
index 5af57e6..6f39f01 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
@@ -22,6 +22,7 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.EOFException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -46,14 +47,14 @@ class S3InputStream extends FSInputStream {
private long pos = 0;
private File blockFile;
-
+
private DataInputStream blockStream;
private long blockEnd = -1;
-
+
private FileSystem.Statistics stats;
-
- private static final Log LOG =
+
+ private static final Log LOG =
LogFactory.getLog(S3InputStream.class.getName());
@@ -65,7 +66,7 @@ class S3InputStream extends FSInputStream {
public S3InputStream(Configuration conf, FileSystemStore store,
INode inode, FileSystem.Statistics stats) {
-
+
this.store = store;
this.stats = stats;
this.blocks = inode.getBlocks();
@@ -86,8 +87,12 @@ class S3InputStream extends FSInputStream {
@Override
public synchronized void seek(long targetPos) throws IOException {
+ String message = String.format("Cannot seek to %d", targetPos);
if (targetPos > fileLength) {
- throw new IOException("Cannot seek after EOF");
+ throw new EOFException(message + ": after EOF");
+ }
+ if (targetPos < 0) {
+ throw new EOFException(message + ": negative");
}
pos = targetPos;
blockEnd = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
new file mode 100644
index 0000000..b388ce7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+public class S3Contract extends AbstractBondedFSContract {
+
+ public static final String CONTRACT_XML = "contract/s3.xml";
+
+
+ public S3Contract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return "s3";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
new file mode 100644
index 0000000..b8c758c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractCreate.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
new file mode 100644
index 0000000..2d3cec7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractDelete.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3ContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
new file mode 100644
index 0000000..992ce53
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractMkdir.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
new file mode 100644
index 0000000..2a4ba31
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractOpen.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
new file mode 100644
index 0000000..68bdbda
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRename.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3ContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
new file mode 100644
index 0000000..b968081
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractRootDir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * root dir operations against an S3 bucket
+ */
+public class TestS3ContractRootDir extends AbstractContractRootDirectoryTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
new file mode 100644
index 0000000..bfcd163
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/TestS3ContractSeek.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+public class TestS3ContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1acc509b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
new file mode 100644
index 0000000..4b742c1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3.xml
@@ -0,0 +1,104 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ S3 is backed by a blobstore.
+ -->
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-available-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+</configuration>
[35/50] [abbrv] hadoop git commit: HDFS-8898. Create API and
command-line argument to get quota and quota usage without detailed content
summary. Contributed by Ming Ma.
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index 961ae0e..9798787 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -134,6 +135,7 @@ public class TestQuota {
// 4: count -q /test
ContentSummary c = dfs.getContentSummary(parent);
+ compareQuotaUsage(c, dfs, parent);
assertEquals(c.getFileCount()+c.getDirectoryCount(), 3);
assertEquals(c.getQuota(), 3);
assertEquals(c.getSpaceConsumed(), fileLen*replication);
@@ -141,10 +143,12 @@ public class TestQuota {
// 5: count -q /test/data0
c = dfs.getContentSummary(childDir0);
+ compareQuotaUsage(c, dfs, childDir0);
assertEquals(c.getFileCount()+c.getDirectoryCount(), 1);
assertEquals(c.getQuota(), -1);
// check disk space consumed
c = dfs.getContentSummary(parent);
+ compareQuotaUsage(c, dfs, parent);
assertEquals(c.getSpaceConsumed(), fileLen*replication);
// 6: create a directory /test/data1
@@ -172,12 +176,14 @@ public class TestQuota {
// 8: clear quota /test
runCommand(admin, new String[]{"-clrQuota", parent.toString()}, false);
c = dfs.getContentSummary(parent);
+ compareQuotaUsage(c, dfs, parent);
assertEquals(c.getQuota(), -1);
assertEquals(c.getSpaceQuota(), spaceQuota);
// 9: clear quota /test/data0
runCommand(admin, new String[]{"-clrQuota", childDir0.toString()}, false);
c = dfs.getContentSummary(childDir0);
+ compareQuotaUsage(c, dfs, childDir0);
assertEquals(c.getQuota(), -1);
// 10: create a file /test/datafile1
@@ -198,6 +204,7 @@ public class TestQuota {
// 9.s: clear diskspace quota
runCommand(admin, false, "-clrSpaceQuota", parent.toString());
c = dfs.getContentSummary(parent);
+ compareQuotaUsage(c, dfs, parent);
assertEquals(c.getQuota(), -1);
assertEquals(c.getSpaceQuota(), -1);
@@ -224,6 +231,7 @@ public class TestQuota {
}
assertTrue(hasException);
c = dfs.getContentSummary(childDir0);
+ compareQuotaUsage(c, dfs, childDir0);
assertEquals(c.getDirectoryCount()+c.getFileCount(), 1);
assertEquals(c.getQuota(), 1);
@@ -362,7 +370,7 @@ public class TestQuota {
}
assertTrue(hasException);
- assertEquals(4, cluster.getNamesystem().getFSDirectory().getYieldCount());
+ assertEquals(5, cluster.getNamesystem().getFSDirectory().getYieldCount());
} finally {
cluster.shutdown();
}
@@ -387,6 +395,7 @@ public class TestQuota {
final Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.setQuota(quotaDir1, 6, HdfsConstants.QUOTA_DONT_SET);
ContentSummary c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 3);
assertEquals(c.getQuota(), 6);
@@ -394,6 +403,7 @@ public class TestQuota {
final Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20");
dfs.setQuota(quotaDir2, 7, HdfsConstants.QUOTA_DONT_SET);
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 2);
assertEquals(c.getQuota(), 7);
@@ -402,6 +412,7 @@ public class TestQuota {
assertTrue(dfs.mkdirs(quotaDir3));
dfs.setQuota(quotaDir3, 2, HdfsConstants.QUOTA_DONT_SET);
c = dfs.getContentSummary(quotaDir3);
+ compareQuotaUsage(c, dfs, quotaDir3);
assertEquals(c.getDirectoryCount(), 1);
assertEquals(c.getQuota(), 2);
@@ -409,6 +420,7 @@ public class TestQuota {
Path tempPath = new Path(quotaDir3, "nqdir32");
assertTrue(dfs.mkdirs(tempPath));
c = dfs.getContentSummary(quotaDir3);
+ compareQuotaUsage(c, dfs, quotaDir3);
assertEquals(c.getDirectoryCount(), 2);
assertEquals(c.getQuota(), 2);
@@ -422,6 +434,7 @@ public class TestQuota {
}
assertTrue(hasException);
c = dfs.getContentSummary(quotaDir3);
+ compareQuotaUsage(c, dfs, quotaDir3);
assertEquals(c.getDirectoryCount(), 2);
assertEquals(c.getQuota(), 2);
@@ -429,9 +442,11 @@ public class TestQuota {
tempPath = new Path(quotaDir2, "nqdir31");
assertTrue(dfs.mkdirs(tempPath));
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 3);
assertEquals(c.getQuota(), 7);
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 6);
assertEquals(c.getQuota(), 6);
@@ -449,9 +464,11 @@ public class TestQuota {
tempPath = new Path(quotaDir2, "nqdir30");
dfs.rename(new Path(quotaDir3, "nqdir32"), tempPath);
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 4);
assertEquals(c.getQuota(), 7);
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 6);
assertEquals(c.getQuota(), 6);
@@ -480,9 +497,11 @@ public class TestQuota {
// 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0
assertTrue(dfs.rename(tempPath, new Path("/nqdir0")));
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 2);
assertEquals(c.getQuota(), 7);
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 4);
assertEquals(c.getQuota(), 6);
@@ -501,31 +520,38 @@ public class TestQuota {
// 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20
assertTrue(dfs.rename(quotaDir3, quotaDir2));
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 4);
assertEquals(c.getQuota(), 6);
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 3);
assertEquals(c.getQuota(), 7);
tempPath = new Path(quotaDir2, "qdir21");
c = dfs.getContentSummary(tempPath);
+ compareQuotaUsage(c, dfs, tempPath);
assertEquals(c.getDirectoryCount(), 1);
assertEquals(c.getQuota(), 2);
// 15: Delete /nqdir0/qdir1/qdir20/qdir21
dfs.delete(tempPath, true);
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 2);
assertEquals(c.getQuota(), 7);
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 3);
assertEquals(c.getQuota(), 6);
// 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20
assertTrue(dfs.rename(new Path("/nqdir0/nqdir30"), quotaDir2));
c = dfs.getContentSummary(quotaDir2);
+ compareQuotaUsage(c, dfs, quotaDir2);
assertEquals(c.getDirectoryCount(), 5);
assertEquals(c.getQuota(), 7);
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getDirectoryCount(), 6);
assertEquals(c.getQuota(), 6);
assertEquals(14, cluster.getNamesystem().getFSDirectory().getYieldCount());
@@ -567,12 +593,14 @@ public class TestQuota {
final Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.setQuota(quotaDir1, HdfsConstants.QUOTA_DONT_SET, 4 * fileSpace);
ContentSummary c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getSpaceQuota(), 4 * fileSpace);
// set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace
final Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
dfs.setQuota(quotaDir20, HdfsConstants.QUOTA_DONT_SET, 6 * fileSpace);
c = dfs.getContentSummary(quotaDir20);
+ compareQuotaUsage(c, dfs, quotaDir20);
assertEquals(c.getSpaceQuota(), 6 * fileSpace);
// Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
@@ -580,6 +608,7 @@ public class TestQuota {
assertTrue(dfs.mkdirs(quotaDir21));
dfs.setQuota(quotaDir21, HdfsConstants.QUOTA_DONT_SET, 2 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
+ compareQuotaUsage(c, dfs, quotaDir21);
assertEquals(c.getSpaceQuota(), 2 * fileSpace);
// 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
@@ -590,6 +619,7 @@ public class TestQuota {
DFSTestUtil.createFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen,
replication, 0);
c = dfs.getContentSummary(quotaDir21);
+ compareQuotaUsage(c, dfs, quotaDir21);
assertEquals(c.getSpaceConsumed(), fileSpace);
// Create a larger file /nqdir0/qdir1/qdir21/nqdir33/
@@ -604,11 +634,13 @@ public class TestQuota {
// delete nqdir33
assertTrue(dfs.delete(new Path(quotaDir21, "nqdir33"), true));
c = dfs.getContentSummary(quotaDir21);
+ compareQuotaUsage(c, dfs, quotaDir21);
assertEquals(c.getSpaceConsumed(), fileSpace);
assertEquals(c.getSpaceQuota(), 2*fileSpace);
// Verify space before the move:
c = dfs.getContentSummary(quotaDir20);
+ compareQuotaUsage(c, dfs, quotaDir20);
assertEquals(c.getSpaceConsumed(), 0);
// Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
@@ -621,9 +653,11 @@ public class TestQuota {
assertEquals(c.getSpaceConsumed(), fileSpace);
// verify space for its parent
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getSpaceConsumed(), fileSpace);
// verify space for source for the move
c = dfs.getContentSummary(quotaDir21);
+ compareQuotaUsage(c, dfs, quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
final Path file2 = new Path(dstPath, "fileDir/file2");
@@ -634,6 +668,7 @@ public class TestQuota {
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
+ compareQuotaUsage(c, dfs, quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
// Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
@@ -654,16 +689,19 @@ public class TestQuota {
c = dfs.getContentSummary(quotaDir20);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
c = dfs.getContentSummary(quotaDir21);
+ compareQuotaUsage(c, dfs, quotaDir21);
assertEquals(c.getSpaceConsumed(), 0);
// Test Append :
// verify space quota
c = dfs.getContentSummary(quotaDir1);
+ compareQuotaUsage(c, dfs, quotaDir1);
assertEquals(c.getSpaceQuota(), 4 * fileSpace);
// verify space before append;
c = dfs.getContentSummary(dstPath);
+ compareQuotaUsage(c, dfs, dstPath);
assertEquals(c.getSpaceConsumed(), 3 * fileSpace);
OutputStream out = dfs.append(file2);
@@ -675,6 +713,7 @@ public class TestQuota {
// verify space after append;
c = dfs.getContentSummary(dstPath);
+ compareQuotaUsage(c, dfs, dstPath);
assertEquals(c.getSpaceConsumed(), 4 * fileSpace);
// now increase the quota for quotaDir1
@@ -696,6 +735,7 @@ public class TestQuota {
// verify space after partial append
c = dfs.getContentSummary(dstPath);
+ compareQuotaUsage(c, dfs, dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace);
// Test set replication :
@@ -705,6 +745,7 @@ public class TestQuota {
// verify that space is reduced by file2Len
c = dfs.getContentSummary(dstPath);
+ compareQuotaUsage(c, dfs, dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now try to increase the replication and and expect an error.
@@ -718,6 +759,7 @@ public class TestQuota {
// verify space consumed remains unchanged.
c = dfs.getContentSummary(dstPath);
+ compareQuotaUsage(c, dfs, dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace - file2Len);
// now increase the quota for quotaDir1 and quotaDir20
@@ -728,6 +770,7 @@ public class TestQuota {
dfs.setReplication(file2, (short)(replication+1));
// verify increase in space
c = dfs.getContentSummary(dstPath);
+ compareQuotaUsage(c, dfs, dstPath);
assertEquals(c.getSpaceConsumed(), 5 * fileSpace + file2Len);
// Test HDFS-2053 :
@@ -755,32 +798,37 @@ public class TestQuota {
dfs.setQuota(quotaDir2053_C, HdfsConstants.QUOTA_DONT_SET,
(sizeFactorC + 1) * fileSpace);
c = dfs.getContentSummary(quotaDir2053_C);
+ compareQuotaUsage(c, dfs, quotaDir2053_C);
assertEquals(c.getSpaceQuota(), (sizeFactorC + 1) * fileSpace);
// Create a file under subdirectory A
DFSTestUtil.createFile(dfs, new Path(quotaDir2053_A, "fileA"),
sizeFactorA * fileLen, replication, 0);
c = dfs.getContentSummary(quotaDir2053_A);
+ compareQuotaUsage(c, dfs, quotaDir2053_A);
assertEquals(c.getSpaceConsumed(), sizeFactorA * fileSpace);
// Create a file under subdirectory B
DFSTestUtil.createFile(dfs, new Path(quotaDir2053_B, "fileB"),
sizeFactorB * fileLen, replication, 0);
c = dfs.getContentSummary(quotaDir2053_B);
+ compareQuotaUsage(c, dfs, quotaDir2053_B);
assertEquals(c.getSpaceConsumed(), sizeFactorB * fileSpace);
// Create a file under subdirectory C (which has a space quota)
DFSTestUtil.createFile(dfs, new Path(quotaDir2053_C, "fileC"),
sizeFactorC * fileLen, replication, 0);
c = dfs.getContentSummary(quotaDir2053_C);
+ compareQuotaUsage(c, dfs, quotaDir2053_C);
assertEquals(c.getSpaceConsumed(), sizeFactorC * fileSpace);
// Check space consumed for /hdfs-2053
c = dfs.getContentSummary(quotaDir2053);
+ compareQuotaUsage(c, dfs, quotaDir2053);
assertEquals(c.getSpaceConsumed(),
(sizeFactorA + sizeFactorB + sizeFactorC) * fileSpace);
- assertEquals(20, cluster.getNamesystem().getFSDirectory().getYieldCount());
+ assertEquals(28, cluster.getNamesystem().getFSDirectory().getYieldCount());
} finally {
cluster.shutdown();
}
@@ -864,21 +912,25 @@ public class TestQuota {
// setting namespace quota to Long.MAX_VALUE - 1 should work
dfs.setQuota(testFolder, Long.MAX_VALUE - 1, 10);
ContentSummary c = dfs.getContentSummary(testFolder);
+ compareQuotaUsage(c, dfs, testFolder);
assertTrue("Quota not set properly", c.getQuota() == Long.MAX_VALUE - 1);
// setting diskspace quota to Long.MAX_VALUE - 1 should work
dfs.setQuota(testFolder, 10, Long.MAX_VALUE - 1);
c = dfs.getContentSummary(testFolder);
+ compareQuotaUsage(c, dfs, testFolder);
assertTrue("Quota not set properly", c.getSpaceQuota() == Long.MAX_VALUE - 1);
// setting namespace quota to Long.MAX_VALUE should not work + no error
dfs.setQuota(testFolder, Long.MAX_VALUE, 10);
c = dfs.getContentSummary(testFolder);
+ compareQuotaUsage(c, dfs, testFolder);
assertTrue("Quota should not have changed", c.getQuota() == 10);
// setting diskspace quota to Long.MAX_VALUE should not work + no error
dfs.setQuota(testFolder, 10, Long.MAX_VALUE);
c = dfs.getContentSummary(testFolder);
+ compareQuotaUsage(c, dfs, testFolder);
assertTrue("Quota should not have changed", c.getSpaceQuota() == 10);
// setting namespace quota to Long.MAX_VALUE + 1 should not work + error
@@ -942,6 +994,7 @@ public class TestQuota {
DFSTestUtil.createFile(fs, file1, FILE_SIZE, (short) 3, 1L);
DFSTestUtil.waitReplication(fs, file1, (short) 3);
c = fs.getContentSummary(dir);
+ compareQuotaUsage(c, fs, dir);
checkContentSummary(c, webhdfs.getContentSummary(dir));
assertEquals("Quota is half consumed", QUOTA_SIZE / 2,
c.getSpaceConsumed());
@@ -1027,6 +1080,7 @@ public class TestQuota {
// Should account for all 59 files (almost QUOTA_SIZE)
c = fs.getContentSummary(dir);
+ compareQuotaUsage(c, fs, dir);
checkContentSummary(c, webhdfs.getContentSummary(dir));
assertEquals("Invalid space consumed", 59 * FILE_SIZE * 3,
c.getSpaceConsumed());
@@ -1088,6 +1142,7 @@ public class TestQuota {
FSDataOutputStream out = dfs.create(new Path("/Folder2/file6"),(short)1);
out.close();
ContentSummary contentSummary = dfs.getContentSummary(new Path("/"));
+ compareQuotaUsage(contentSummary, dfs, new Path("/"));
assertEquals(6, contentSummary.getFileCount());
} finally {
if (cluster != null) {
@@ -1097,4 +1152,11 @@ public class TestQuota {
}
}
+ // check the QuotaUsage got from getContentSummary is the same as
+ // getQuotaUsage
+ private void compareQuotaUsage(final QuotaUsage fromContentSummary,
+ final FileSystem fileSystem, final Path filePath) throws IOException {
+ QuotaUsage quotaUsage = fileSystem.getQuotaUsage(filePath);
+ assertEquals(fromContentSummary, quotaUsage);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
index e768306..cbd01b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestQuotasWithHA.java
@@ -139,11 +139,23 @@ public class TestQuotasWithHA {
* exception.
*/
@Test(expected = StandbyException.class)
- public void testgetContentSummaryOnStandby() throws Exception {
+ public void testGetContentSummaryOnStandby() throws Exception {
Configuration nn1conf =cluster.getConfiguration(1);
// just reset the standby reads to default i.e False on standby.
HAUtil.setAllowStandbyReads(nn1conf, false);
cluster.restartNameNode(1);
cluster.getNameNodeRpc(1).getContentSummary("/");
}
+
+ /**
+ * Test that getQuotaUsage on Standby should should throw standby exception.
+ */
+ @Test(expected = StandbyException.class)
+ public void testGetQuotaUsageOnStandby() throws Exception {
+ Configuration nn1conf =cluster.getConfiguration(1);
+ // just reset the standby reads to default i.e False on standby.
+ HAUtil.setAllowStandbyReads(nn1conf, false);
+ cluster.restartNameNode(1);
+ cluster.getNameNodeRpc(1).getQuotaUsage("/");
+ }
}
[42/50] [abbrv] hadoop git commit: YARN-4496. Improve HA
ResourceManager Failover detection on the client. Contributed by Jian He
Posted by ka...@apache.org.
YARN-4496. Improve HA ResourceManager Failover detection on the client.
Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/618bfd6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/618bfd6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/618bfd6a
Branch: refs/heads/YARN-1011
Commit: 618bfd6ac2a5b62d39e9bed80f75362bafc0ef28
Parents: 46e5ea8
Author: Xuan <xg...@apache.org>
Authored: Fri Jan 22 18:20:38 2016 -0800
Committer: Xuan <xg...@apache.org>
Committed: Fri Jan 22 18:20:38 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
...stHedgingRequestRMFailoverProxyProvider.java | 98 ++++++++++
.../ConfiguredRMFailoverProxyProvider.java | 6 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 33 ++--
.../RequestHedgingRMFailoverProxyProvider.java | 194 +++++++++++++++++++
.../nodemanager/TestNodeStatusUpdater.java | 7 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 1 +
7 files changed, 323 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f840a9e..8e87f4a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -104,6 +104,9 @@ Release 2.9.0 - UNRELEASED
YARN-4603. FairScheduler should mention user requested queuename in error
message when failed in queue ACL check. (Tao Jie via kasha)
+ YARN-4496. Improve HA ResourceManager Failover detection on the client.
+ (Jian He via xgong)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java
new file mode 100644
index 0000000..6fd6591
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestHedgingRequestRMFailoverProxyProvider.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestHedgingRequestRMFailoverProxyProvider {
+
+ @Test
+ public void testHedgingRequestProxyProvider() throws Exception {
+ final MiniYARNCluster cluster =
+ new MiniYARNCluster("testHedgingRequestProxyProvider", 5, 0, 1, 1);
+ Configuration conf = new YarnConfiguration();
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3,rm4,rm5");
+
+ conf.set(YarnConfiguration.CLIENT_FAILOVER_PROXY_PROVIDER,
+ RequestHedgingRMFailoverProxyProvider.class.getName());
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ 2000);
+
+ HATestUtil.setRpcAddressForRM("rm1", 10000, conf);
+ HATestUtil.setRpcAddressForRM("rm2", 20000, conf);
+ HATestUtil.setRpcAddressForRM("rm3", 30000, conf);
+ HATestUtil.setRpcAddressForRM("rm4", 40000, conf);
+ HATestUtil.setRpcAddressForRM("rm5", 50000, conf);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+
+ cluster.init(conf);
+ cluster.start();
+
+ final YarnClient client = YarnClient.createYarnClient();
+ client.init(conf);
+ client.start();
+
+ // Transition rm5 to active;
+ long start = System.currentTimeMillis();
+ makeRMActive(cluster, 4);
+ // client will retry until the rm becomes active.
+ client.getAllQueues();
+ long end = System.currentTimeMillis();
+ System.out.println("Client call succeeded at " + end);
+ // should return the response fast
+ Assert.assertTrue(end - start <= 10000);
+
+ // transition rm5 to standby
+ cluster.getResourceManager(4).getRMContext().getRMAdminService()
+ .transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER));
+
+ makeRMActive(cluster, 2);
+ client.getAllQueues();
+ cluster.stop();
+ }
+
+ private void makeRMActive(final MiniYARNCluster cluster, final int index) {
+ Thread t = new Thread() {
+ @Override public void run() {
+ try {
+ System.out.println("Transition rm" + index + " to active");
+ cluster.getResourceManager(index).getRMContext().getRMAdminService()
+ .transitionToActive(new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ };
+ t.start();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
index 5577d20..8676db2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ConfiguredRMFailoverProxyProvider.java
@@ -45,8 +45,8 @@ public class ConfiguredRMFailoverProxyProvider<T>
private int currentProxyIndex = 0;
Map<String, T> proxies = new HashMap<String, T>();
- private RMProxy<T> rmProxy;
- private Class<T> protocol;
+ protected RMProxy<T> rmProxy;
+ protected Class<T> protocol;
protected YarnConfiguration conf;
protected String[] rmServiceIds;
@@ -71,7 +71,7 @@ public class ConfiguredRMFailoverProxyProvider<T>
YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
}
- private T getProxyInternal() {
+ protected T getProxyInternal() {
try {
final InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
return RMProxy.getProxy(conf, protocol, rmAddress);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index 3779ce5..3ab06bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
@@ -77,6 +78,7 @@ public class RMProxy<T> {
}
/**
+ * Currently, used by Client and AM only
* Create a proxy for the specified protocol. For non-HA,
* this is a direct connection to the ResourceManager address. When HA is
* enabled, the proxy handles the failover between the ResourceManagers as
@@ -88,12 +90,12 @@ public class RMProxy<T> {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
: new YarnConfiguration(configuration);
- RetryPolicy retryPolicy =
- createRetryPolicy(conf);
- return createRMProxy(conf, protocol, instance, retryPolicy);
+ RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
+ return newProxyInstance(conf, protocol, instance, retryPolicy);
}
/**
+ * Currently, used by NodeManagers only.
* Create a proxy for the specified protocol. For non-HA,
* this is a direct connection to the ResourceManager address. When HA is
* enabled, the proxy handles the failover between the ResourceManagers as
@@ -106,12 +108,12 @@ public class RMProxy<T> {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
: new YarnConfiguration(configuration);
- RetryPolicy retryPolicy =
- createRetryPolicy(conf, retryTime, retryInterval);
- return createRMProxy(conf, protocol, instance, retryPolicy);
+ RetryPolicy retryPolicy = createRetryPolicy(conf, retryTime, retryInterval,
+ HAUtil.isHAEnabled(conf));
+ return newProxyInstance(conf, protocol, instance, retryPolicy);
}
- private static <T> T createRMProxy(final YarnConfiguration conf,
+ private static <T> T newProxyInstance(final YarnConfiguration conf,
final Class<T> protocol, RMProxy instance, RetryPolicy retryPolicy)
throws IOException{
if (HAUtil.isHAEnabled(conf)) {
@@ -144,7 +146,7 @@ public class RMProxy<T> {
@Deprecated
public static <T> T createRMProxy(final Configuration conf,
final Class<T> protocol, InetSocketAddress rmAddress) throws IOException {
- RetryPolicy retryPolicy = createRetryPolicy(conf);
+ RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
LOG.info("Connecting to ResourceManager at " + rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
@@ -194,7 +196,8 @@ public class RMProxy<T> {
*/
@Private
@VisibleForTesting
- public static RetryPolicy createRetryPolicy(Configuration conf) {
+ public static RetryPolicy createRetryPolicy(Configuration conf,
+ boolean isHAEnabled) {
long rmConnectWaitMS =
conf.getLong(
YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
@@ -204,16 +207,17 @@ public class RMProxy<T> {
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
YarnConfiguration
.DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);
- return createRetryPolicy(
- conf, rmConnectWaitMS, rmConnectionRetryIntervalMS);
+
+ return createRetryPolicy(conf, rmConnectWaitMS, rmConnectionRetryIntervalMS,
+ isHAEnabled);
}
/**
* Fetch retry policy from Configuration and create the
* retry policy with specified retryTime and retry interval.
*/
- private static RetryPolicy createRetryPolicy(Configuration conf,
- long retryTime, long retryInterval) {
+ protected static RetryPolicy createRetryPolicy(Configuration conf,
+ long retryTime, long retryInterval, boolean isHAEnabled) {
long rmConnectWaitMS = retryTime;
long rmConnectionRetryIntervalMS = retryInterval;
@@ -236,7 +240,7 @@ public class RMProxy<T> {
}
// Handle HA case first
- if (HAUtil.isHAEnabled(conf)) {
+ if (isHAEnabled) {
final long failoverSleepBaseMs = conf.getLong(
YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
rmConnectionRetryIntervalMS);
@@ -287,6 +291,7 @@ public class RMProxy<T> {
exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
exceptionToPolicyMap.put(SocketException.class, retryPolicy);
+ exceptionToPolicyMap.put(StandbyException.class, retryPolicy);
// YARN-4288: local IOException is also possible.
exceptionToPolicyMap.put(IOException.class, retryPolicy);
// Not retry on remote IO exception.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
new file mode 100644
index 0000000..dc8d19b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RequestHedgingRMFailoverProxyProvider.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.MultiException;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * A FailoverProxyProvider implementation that technically does not "failover"
+ * per-se. It constructs a wrapper proxy that sends the request to ALL
+ * underlying proxies simultaneously. Each proxy inside the wrapper proxy will
+ * retry the corresponding target. It assumes the in an HA setup, there will
+ * be only one Active, and the active should respond faster than any configured
+ * standbys. Once it receives a response from any one of the configred proxies,
+ * outstanding requests to other proxies are immediately cancelled.
+ */
+public class RequestHedgingRMFailoverProxyProvider<T>
+ extends ConfiguredRMFailoverProxyProvider<T> {
+
+ private static final Log LOG =
+ LogFactory.getLog(RequestHedgingRMFailoverProxyProvider.class);
+
+ private volatile String successfulProxy = null;
+ private ProxyInfo<T> wrappedProxy = null;
+ private Map<String, T> nonRetriableProxy = new HashMap<>();
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(Configuration configuration, RMProxy<T> rmProxy,
+ Class<T> protocol) {
+ super.init(configuration, rmProxy, protocol);
+ Map<String, ProxyInfo<T>> retriableProxies = new HashMap<>();
+
+ String originalId = HAUtil.getRMHAId(conf);
+ for (String rmId : rmServiceIds) {
+ conf.set(YarnConfiguration.RM_HA_ID, rmId);
+ nonRetriableProxy.put(rmId, super.getProxyInternal());
+
+ T proxy = createRetriableProxy();
+ ProxyInfo<T> pInfo = new ProxyInfo<T>(proxy, rmId);
+ retriableProxies.put(rmId, pInfo);
+ }
+ conf.set(YarnConfiguration.RM_HA_ID, originalId);
+
+ T proxyInstance = (T) Proxy.newProxyInstance(
+ RMRequestHedgingInvocationHandler.class.getClassLoader(),
+ new Class<?>[] {protocol},
+ new RMRequestHedgingInvocationHandler(retriableProxies));
+ String combinedInfo = Arrays.toString(rmServiceIds);
+ wrappedProxy = new ProxyInfo<T>(proxyInstance, combinedInfo);
+ LOG.info("Created wrapped proxy for " + combinedInfo);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected T createRetriableProxy() {
+ try {
+ // Create proxy that can retry exceptions properly.
+ RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf, false);
+ InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
+ T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
+ return (T) RetryProxy.create(protocol, proxy, retryPolicy);
+ } catch (IOException ioe) {
+ LOG.error("Unable to create proxy to the ResourceManager " + HAUtil
+ .getRMHAId(conf), ioe);
+ return null;
+ }
+ }
+
+ class RMRequestHedgingInvocationHandler implements InvocationHandler {
+
+ final private Map<String, ProxyInfo<T>> allProxies;
+
+ public RMRequestHedgingInvocationHandler(
+ Map<String, ProxyInfo<T>> allProxies) {
+ this.allProxies = new HashMap<>(allProxies);
+ }
+
+ protected Object invokeMethod(Object proxy, Method method, Object[] args)
+ throws Throwable {
+ try {
+ return method.invoke(proxy, args);
+ } catch (InvocationTargetException ex) {
+ throw ex.getCause();
+ }
+ }
+
+ /**
+ * Creates a Executor and invokes all proxies concurrently.
+ */
+ @Override
+ public Object invoke(Object proxy, final Method method,
+ final Object[] args) throws Throwable {
+ if (successfulProxy != null) {
+ return invokeMethod(nonRetriableProxy.get(successfulProxy), method, args);
+ }
+
+ ExecutorService executor = null;
+ CompletionService<Object> completionService;
+ try {
+ Map<Future<Object>, ProxyInfo<T>> proxyMap = new HashMap<>();
+ int numAttempts = 0;
+ executor = Executors.newFixedThreadPool(allProxies.size());
+ completionService = new ExecutorCompletionService<>(executor);
+ for (final ProxyInfo<T> pInfo : allProxies.values()) {
+ Callable<Object> c = new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return method.invoke(pInfo.proxy, args);
+ }
+ };
+ proxyMap.put(completionService.submit(c), pInfo);
+ numAttempts++;
+ }
+
+ Map<String, Exception> badResults = new HashMap<>();
+ while (numAttempts > 0) {
+ Future<Object> callResultFuture = completionService.take();
+ String pInfo = proxyMap.get(callResultFuture).proxyInfo;
+ Object retVal;
+ try {
+ retVal = callResultFuture.get();
+ successfulProxy = pInfo;
+ LOG.info("Invocation successful on [" + pInfo + "]");
+ return retVal;
+ } catch (Exception ex) {
+ LOG.warn("Invocation returned exception on " + "[" + pInfo + "]");
+ badResults.put(pInfo, ex);
+ numAttempts--;
+ }
+ }
+
+ // At this point we should have All bad results (Exceptions)
+ // Or should have returned with successful result.
+ if (badResults.size() == 1) {
+ throw badResults.values().iterator().next();
+ } else {
+ throw new MultiException(badResults);
+ }
+ } finally {
+ if (executor != null) {
+ executor.shutdownNow();
+ }
+ }
+ }
+ }
+
+ @Override
+ public ProxyInfo<T> getProxy() {
+ return wrappedProxy;
+ }
+
+ @Override
+ public void performFailover(T currentProxy) {
+ LOG.info("Connection lost, trying to fail over.");
+ successfulProxy = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 90804b8..a8066c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.client.RMProxy;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -443,7 +444,8 @@ public class TestNodeStatusUpdater {
@Override
protected ResourceTracker getRMClient() throws IOException {
- RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf);
+ RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf,
+ HAUtil.isHAEnabled(conf));
resourceTracker =
(ResourceTracker) RetryProxy.create(ResourceTracker.class,
new MyResourceTracker6(rmStartIntervalMS, rmNeverStart),
@@ -476,7 +478,8 @@ public class TestNodeStatusUpdater {
@Override
protected ResourceTracker getRMClient() {
- RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf);
+ RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf,
+ HAUtil.isHAEnabled(conf));
return (ResourceTracker) RetryProxy.create(ResourceTracker.class,
resourceTracker, retryPolicy);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/618bfd6a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 68c9efd..630b7ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -457,6 +457,7 @@ public class MiniYARNCluster extends CompositeService {
protected synchronized void serviceStart() throws Exception {
startResourceManager(index);
Configuration conf = resourceManagers[index].getConfig();
+ LOG.info("Starting resourcemanager " + index);
LOG.info("MiniYARN ResourceManager address: " +
conf.get(YarnConfiguration.RM_ADDRESS));
LOG.info("MiniYARN ResourceManager web address: " + WebAppUtils
[08/50] [abbrv] hadoop git commit: YARN-4559. Make leader elector and
zk store share the same curator client. Contributed by Jian He
Posted by ka...@apache.org.
YARN-4559. Make leader elector and zk store share the same curator
client. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/890a2ebd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/890a2ebd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/890a2ebd
Branch: refs/heads/YARN-1011
Commit: 890a2ebd1af51d24ccbbc5d1d65d17b24ad8ab9b
Parents: 2ec438e
Author: Xuan <xg...@apache.org>
Authored: Wed Jan 20 14:48:10 2016 -0800
Committer: Xuan <xg...@apache.org>
Committed: Wed Jan 20 14:48:10 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 5 +
.../resourcemanager/LeaderElectorService.java | 23 +---
.../server/resourcemanager/ResourceManager.java | 98 +++++++++++++---
.../resourcemanager/recovery/RMStateStore.java | 2 +-
.../recovery/ZKRMStateStore.java | 111 +++++--------------
.../recovery/TestZKRMStateStore.java | 1 +
.../recovery/TestZKRMStateStorePerf.java | 2 +
.../TestZKRMStateStoreZKClientConnections.java | 22 +---
9 files changed, 128 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 29ee57b..ee78d22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -98,6 +98,9 @@ Release 2.9.0 - UNRELEASED
YARN-4526. Make SystemClock singleton so AppSchedulingInfo could use it.
(kasha)
+ YARN-4559. Make leader elector and zk store share the same curator client.
+ (Jian He via xgong)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 6fb3945..c12377b 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -272,6 +272,11 @@
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore" />
+ <Field name="resourceManager"/>
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer"/>
<Field name="renewalTimer" />
<Bug code="IS"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/LeaderElectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/LeaderElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/LeaderElectorService.java
index 3766676..8c1a6eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/LeaderElectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/LeaderElectorService.java
@@ -19,14 +19,11 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.leader.LeaderLatch;
import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
-import org.apache.curator.retry.RetryNTimes;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.service.AbstractService;
@@ -44,35 +41,23 @@ public class LeaderElectorService extends AbstractService implements
private RMContext rmContext;
private String latchPath;
private String rmId;
+ private ResourceManager rm;
- public LeaderElectorService(RMContext rmContext) {
+ public LeaderElectorService(RMContext rmContext, ResourceManager rm) {
super(LeaderElectorService.class.getName());
this.rmContext = rmContext;
-
+ this.rm = rm;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- String zkHostPort = conf.get(YarnConfiguration.RM_ZK_ADDRESS);
- Preconditions.checkNotNull(zkHostPort,
- YarnConfiguration.RM_ZK_ADDRESS + " is not set");
-
rmId = HAUtil.getRMHAId(conf);
String clusterId = YarnConfiguration.getClusterId(conf);
-
- int zkSessionTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
- YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
- int maxRetryNum = conf.getInt(YarnConfiguration.RM_ZK_NUM_RETRIES,
- YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES);
-
String zkBasePath = conf.get(
YarnConfiguration.AUTO_FAILOVER_ZK_BASE_PATH,
YarnConfiguration.DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH);
latchPath = zkBasePath + "/" + clusterId;
-
- curator = CuratorFrameworkFactory.builder().connectString(zkHostPort)
- .retryPolicy(new RetryNTimes(maxRetryNum, zkSessionTimeout)).build();
- curator.start();
+ curator = rm.getCurator();
initAndStartLeaderLatch();
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 33431f8..40d627e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.curator.framework.AuthInfo;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
@@ -28,7 +32,11 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.security.*;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.AbstractService;
@@ -40,6 +48,7 @@ import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.ZKUtil;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -58,8 +67,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMaste
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
@@ -78,7 +87,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
@@ -96,12 +107,15 @@ import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.WebApps.Builder;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
@@ -158,6 +172,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected ResourceTrackerService resourceTracker;
private JvmPauseMonitor pauseMonitor;
private boolean curatorEnabled = false;
+ private CuratorFramework curator;
+ private final String zkRootNodePassword =
+ Long.toString(new SecureRandom().nextLong());
+ private boolean recoveryEnabled;
@VisibleForTesting
protected String webAppAddress;
@@ -232,7 +250,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
curatorEnabled = conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR,
YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED);
if (curatorEnabled) {
- LeaderElectorService elector = new LeaderElectorService(rmContext);
+ this.curator = createAndStartCurator(conf);
+ LeaderElectorService elector = new LeaderElectorService(rmContext, this);
addService(elector);
rmContext.setLeaderElectorService(elector);
}
@@ -276,7 +295,58 @@ public class ResourceManager extends CompositeService implements Recoverable {
super.serviceInit(this.conf);
}
-
+
+ public CuratorFramework createAndStartCurator(Configuration conf)
+ throws Exception {
+ String zkHostPort = conf.get(YarnConfiguration.RM_ZK_ADDRESS);
+ if (zkHostPort == null) {
+ throw new YarnRuntimeException(
+ YarnConfiguration.RM_ZK_ADDRESS + " is not configured.");
+ }
+ int numRetries = conf.getInt(YarnConfiguration.RM_ZK_NUM_RETRIES,
+ YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES);
+ int zkSessionTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
+ YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
+ int zkRetryInterval = conf.getInt(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS);
+
+ // set up zk auths
+ List<ZKUtil.ZKAuthInfo> zkAuths = RMZKUtils.getZKAuths(conf);
+ List<AuthInfo> authInfos = new ArrayList<>();
+ for (ZKUtil.ZKAuthInfo zkAuth : zkAuths) {
+ authInfos.add(new AuthInfo(zkAuth.getScheme(), zkAuth.getAuth()));
+ }
+
+ if (HAUtil.isHAEnabled(conf) && HAUtil.getConfValueForRMInstance(
+ YarnConfiguration.ZK_RM_STATE_STORE_ROOT_NODE_ACL, conf) == null) {
+ String zkRootNodeUsername = HAUtil
+ .getConfValueForRMInstance(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS, conf);
+ byte[] defaultFencingAuth =
+ (zkRootNodeUsername + ":" + zkRootNodePassword)
+ .getBytes(Charset.forName("UTF-8"));
+ authInfos.add(new AuthInfo(new DigestAuthenticationProvider().getScheme(),
+ defaultFencingAuth));
+ }
+
+ CuratorFramework client = CuratorFrameworkFactory.builder()
+ .connectString(zkHostPort)
+ .sessionTimeoutMs(zkSessionTimeout)
+ .retryPolicy(new RetryNTimes(numRetries, zkRetryInterval))
+ .authorization(authInfos).build();
+ client.start();
+ return client;
+ }
+
+ public CuratorFramework getCurator() {
+ return this.curator;
+ }
+
+ public String getZkRootNodePassword() {
+ return this.zkRootNodePassword;
+ }
+
+
protected QueueACLsManager createQueueACLsManager(ResourceScheduler scheduler,
Configuration conf) {
return new QueueACLsManager(scheduler, conf);
@@ -412,7 +482,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
private ApplicationMasterLauncher applicationMasterLauncher;
private ContainerAllocationExpirer containerAllocationExpirer;
private ResourceManager rm;
- private boolean recoveryEnabled;
private RMActiveServiceContext activeServiceContext;
RMActiveServices(ResourceManager rm) {
@@ -453,29 +522,26 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setRMDelegatedNodeLabelsUpdater(delegatedNodeLabelsUpdater);
}
- boolean isRecoveryEnabled = conf.getBoolean(
- YarnConfiguration.RECOVERY_ENABLED,
+ recoveryEnabled = conf.getBoolean(YarnConfiguration.RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
RMStateStore rmStore = null;
- if (isRecoveryEnabled) {
- recoveryEnabled = true;
+ if (recoveryEnabled) {
rmStore = RMStateStoreFactory.getStore(conf);
boolean isWorkPreservingRecoveryEnabled =
conf.getBoolean(
YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);
rmContext
- .setWorkPreservingRecoveryEnabled(isWorkPreservingRecoveryEnabled);
+ .setWorkPreservingRecoveryEnabled(isWorkPreservingRecoveryEnabled);
} else {
- recoveryEnabled = false;
rmStore = new NullRMStateStore();
}
try {
+ rmStore.setResourceManager(rm);
rmStore.init(conf);
rmStore.setRMDispatcher(rmDispatcher);
- rmStore.setResourceManager(rm);
} catch (Exception e) {
// the Exception from stateStore.init() needs to be handled for
// HA and we need to give up master status if we got fenced
@@ -1130,6 +1196,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
configurationProvider.close();
}
super.serviceStop();
+ if (curator != null) {
+ curator.close();
+ }
transitionToStandby(false);
rmContext.setHAServiceState(HAServiceState.STOPPING);
}
@@ -1177,7 +1246,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
public ClientRMService getClientRMService() {
return this.clientRM;
}
-
+
/**
* return the scheduler.
* @return the scheduler for the Resource Manager.
@@ -1348,5 +1417,4 @@ public class ResourceManager extends CompositeService implements Recoverable {
out.println(" "
+ "[-remove-application-from-state-store <appId>]" + "\n");
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index ae17aaa..159c11c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -95,7 +95,7 @@ public abstract class RMStateStore extends AbstractService {
"ReservationSystemRoot";
protected static final String VERSION_NODE = "RMVersionNode";
protected static final String EPOCH_NODE = "EpochNode";
- private ResourceManager resourceManager;
+ protected ResourceManager resourceManager;
private final ReadLock readLock;
private final WriteLock writeLock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index ddb8a0b..51e5829 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -18,26 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.nio.charset.Charset;
-import java.security.NoSuchAlgorithmException;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.curator.framework.AuthInfo;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.transaction.CuratorTransaction;
import org.apache.curator.framework.api.transaction.CuratorTransactionFinal;
-import org.apache.curator.retry.RetryNTimes;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -77,7 +64,15 @@ import org.apache.zookeeper.data.Id;
import org.apache.zookeeper.data.Stat;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
/**
* {@link RMStateStore} implementation backed by ZooKeeper.
@@ -140,12 +135,6 @@ public class ZKRMStateStore extends RMStateStore {
private static final String RM_DT_MASTER_KEYS_ROOT_ZNODE_NAME =
"RMDTMasterKeysRoot";
- private String zkHostPort = null;
- private int numRetries;
- private int zkSessionTimeout;
- @VisibleForTesting
- int zkRetryInterval;
-
/** Znode paths */
private String zkRootNodePath;
private String rmAppRoot;
@@ -160,17 +149,15 @@ public class ZKRMStateStore extends RMStateStore {
/** Fencing related variables */
private static final String FENCING_LOCK = "RM_ZK_FENCING_LOCK";
- private boolean useDefaultFencingScheme = false;
private String fencingNodePath;
private Thread verifyActiveStatusThread;
+ private int zkSessionTimeout;
/** ACL and auth info */
private List<ACL> zkAcl;
- private List<ZKUtil.ZKAuthInfo> zkAuths;
@VisibleForTesting
List<ACL> zkRootNodeAcl;
private String zkRootNodeUsername;
- private final String zkRootNodePassword = Long.toString(random.nextLong());
public static final int CREATE_DELETE_PERMS =
ZooDefs.Perms.CREATE | ZooDefs.Perms.DELETE;
private final String zkRootNodeAuthScheme =
@@ -204,45 +191,25 @@ public class ZKRMStateStore extends RMStateStore {
YarnConfiguration.DEFAULT_RM_ADDRESS, conf);
Id rmId = new Id(zkRootNodeAuthScheme,
DigestAuthenticationProvider.generateDigest(
- zkRootNodeUsername + ":" + zkRootNodePassword));
+ zkRootNodeUsername + ":" + resourceManager.getZkRootNodePassword()));
zkRootNodeAcl.add(new ACL(CREATE_DELETE_PERMS, rmId));
return zkRootNodeAcl;
}
@Override
public synchronized void initInternal(Configuration conf) throws Exception {
- zkHostPort = conf.get(YarnConfiguration.RM_ZK_ADDRESS);
- if (zkHostPort == null) {
- throw new YarnRuntimeException("No server address specified for " +
- "zookeeper state store for Resource Manager recovery. " +
- YarnConfiguration.RM_ZK_ADDRESS + " is not configured.");
- }
- numRetries =
- conf.getInt(YarnConfiguration.RM_ZK_NUM_RETRIES,
- YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES);
+
+ /* Initialize fencing related paths, acls, and ops */
znodeWorkingPath =
conf.get(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH,
YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH);
- zkSessionTimeout =
- conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
- YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
-
- if (HAUtil.isHAEnabled(conf)) {
- zkRetryInterval = zkSessionTimeout / numRetries;
- } else {
- zkRetryInterval =
- conf.getInt(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS,
- YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS);
- }
-
- zkAcl = RMZKUtils.getZKAcls(conf);
- zkAuths = RMZKUtils.getZKAuths(conf);
-
zkRootNodePath = getNodePath(znodeWorkingPath, ROOT_ZNODE_NAME);
+ fencingNodePath = getNodePath(zkRootNodePath, FENCING_LOCK);
rmAppRoot = getNodePath(zkRootNodePath, RM_APP_ROOT);
+ zkSessionTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
+ YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
- /* Initialize fencing related paths, acls, and ops */
- fencingNodePath = getNodePath(zkRootNodePath, FENCING_LOCK);
+ zkAcl = RMZKUtils.getZKAcls(conf);
if (HAUtil.isHAEnabled(conf)) {
String zkRootNodeAclConf = HAUtil.getConfValueForRMInstance
(YarnConfiguration.ZK_RM_STATE_STORE_ROOT_NODE_ACL, conf);
@@ -256,7 +223,6 @@ public class ZKRMStateStore extends RMStateStore {
throw bafe;
}
} else {
- useDefaultFencingScheme = true;
zkRootNodeAcl = constructZkRootNodeACL(conf, zkAcl);
}
}
@@ -272,19 +238,22 @@ public class ZKRMStateStore extends RMStateStore {
amrmTokenSecretManagerRoot =
getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT);
reservationRoot = getNodePath(zkRootNodePath, RESERVATION_SYSTEM_ROOT);
+ curatorFramework = resourceManager.getCurator();
+ if (curatorFramework == null) {
+ curatorFramework = resourceManager.createAndStartCurator(conf);
+ }
}
@Override
public synchronized void startInternal() throws Exception {
- // createConnection for future API calls
- createConnection();
// ensure root dirs exist
createRootDirRecursively(znodeWorkingPath);
create(zkRootNodePath);
setRootNodeAcls();
delete(fencingNodePath);
- if (HAUtil.isHAEnabled(getConfig())) {
+ if (HAUtil.isHAEnabled(getConfig()) && !HAUtil
+ .isAutomaticFailoverEnabled(getConfig())) {
verifyActiveStatusThread = new VerifyActiveStatusThread();
verifyActiveStatusThread.start();
}
@@ -332,7 +301,9 @@ public class ZKRMStateStore extends RMStateStore {
verifyActiveStatusThread.interrupt();
verifyActiveStatusThread.join(1000);
}
- IOUtils.closeStream(curatorFramework);
+ if (!HAUtil.isHAEnabled(getConfig())) {
+ IOUtils.closeStream(curatorFramework);
+ }
}
@Override
@@ -909,34 +880,6 @@ public class ZKRMStateStore extends RMStateStore {
}
}
- /*
- * ZK operations using curator
- */
- private void createConnection() throws Exception {
- // Curator connection
- CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
- builder = builder.connectString(zkHostPort)
- .connectionTimeoutMs(zkSessionTimeout)
- .retryPolicy(new RetryNTimes(numRetries, zkRetryInterval));
-
- // Set up authorization based on fencing scheme
- List<AuthInfo> authInfos = new ArrayList<>();
- for (ZKUtil.ZKAuthInfo zkAuth : zkAuths) {
- authInfos.add(new AuthInfo(zkAuth.getScheme(), zkAuth.getAuth()));
- }
- if (useDefaultFencingScheme) {
- byte[] defaultFencingAuth =
- (zkRootNodeUsername + ":" + zkRootNodePassword).getBytes(
- Charset.forName("UTF-8"));
- authInfos.add(new AuthInfo(zkRootNodeAuthScheme, defaultFencingAuth));
- }
- builder = builder.authorization(authInfos);
-
- // Connect to ZK
- curatorFramework = builder.build();
- curatorFramework.start();
- }
-
@VisibleForTesting
byte[] getData(final String path) throws Exception {
return curatorFramework.getData().forPath(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 406fcf6..7df31cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -105,6 +105,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
public TestZKRMStateStoreInternal(Configuration conf, String workingZnode)
throws Exception {
+ setResourceManager(new ResourceManager());
init(conf);
start();
assertTrue(znodeWorkingPath.equals(workingZnode));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
index e270404..4b0b06a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
@@ -103,6 +104,7 @@ public class TestZKRMStateStorePerf extends RMStateStoreTestBase
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
store = new ZKRMStateStore();
+ store.setResourceManager(new ResourceManager());
store.init(conf);
store.start();
when(rmContext.getStateStore()).thenReturn(store);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/890a2ebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
index d188450..6b19be3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.curator.test.TestingServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreTestBase.TestDispatcher;
import org.apache.hadoop.util.ZKUtil;
@@ -80,6 +81,7 @@ public class TestZKRMStateStoreZKClientConnections {
public TestZKRMStateStore(Configuration conf, String workingZnode)
throws Exception {
+ setResourceManager(new ResourceManager());
init(conf);
start();
assertTrue(znodeWorkingPath.equals(workingZnode));
@@ -168,24 +170,4 @@ public class TestZKRMStateStoreZKClientConnections {
zkClientTester.getRMStateStore(conf);
}
-
- @Test
- public void testZKRetryInterval() throws Exception {
- TestZKClient zkClientTester = new TestZKClient();
- YarnConfiguration conf = new YarnConfiguration();
-
- ZKRMStateStore store =
- (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
- assertEquals(YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS,
- store.zkRetryInterval);
- store.stop();
-
- conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
- store =
- (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
- assertEquals(YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS /
- YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES,
- store.zkRetryInterval);
- store.stop();
- }
}
[31/50] [abbrv] hadoop git commit: YARN-4605. Spelling mistake in the
help message of "yarn applicationattempt" command. Contributed by Weiwei
Yang.
Posted by ka...@apache.org.
YARN-4605. Spelling mistake in the help message of "yarn applicationattempt" command. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f58f742
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f58f742
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f58f742
Branch: refs/heads/YARN-1011
Commit: 8f58f742aea87b2b46b9741ffeaebfa36af3573f
Parents: f5c8c85
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jan 22 19:43:06 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Jan 22 19:43:06 2016 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java | 4 ++--
.../java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../hadoop/yarn/server/resourcemanager/ClientRMService.java | 6 +++---
.../yarn/server/resourcemanager/TestClientRMService.java | 6 +++---
5 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f58f742/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 77b4eb4..2230b42 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1304,6 +1304,9 @@ Release 2.8.0 - UNRELEASED
YARN-4608. Redundant code statement in WritingYarnApplications.
(Kai Sasaki via aajisaka)
+ YARN-4605. Spelling mistake in the help message of "yarn applicationattempt"
+ command. (Weiwei Yang via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f58f742/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index bdd6215..77e0688 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -138,7 +138,7 @@ public class ApplicationCLI extends YarnCLI {
opts.addOption(STATUS_CMD, true,
"Prints the status of the application attempt.");
opts.addOption(LIST_CMD, true,
- "List application attempts for aplication.");
+ "List application attempts for application.");
opts.addOption(FAIL_CMD, true, "Fails application attempt.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
@@ -712,7 +712,7 @@ public class ApplicationCLI extends YarnCLI {
throws YarnException, IOException {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
- sysout.println("Updating priority of an aplication " + applicationId);
+ sysout.println("Updating priority of an application " + applicationId);
Priority updateApplicationPriority =
client.updateApplicationPriority(appId, newAppPriority);
if (newAppPriority.equals(updateApplicationPriority)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f58f742/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 1a77c7c..95ccccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1721,7 +1721,7 @@ public class TestYarnCLI {
pw.println(" -fail <Application Attempt ID> Fails application attempt.");
pw.println(" -help Displays help for all commands.");
pw.println(" -list <Application ID> List application attempts for");
- pw.println(" aplication.");
+ pw.println(" application.");
pw.println(" -status <Application Attempt ID> Prints the status of the application");
pw.println(" attempt.");
pw.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f58f742/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 6b0a756..199f098 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -439,7 +439,7 @@ public class ClientRMService extends AbstractService implements
response = GetApplicationAttemptsResponse.newInstance(listAttempts);
} else {
throw new YarnException("User " + callerUGI.getShortUserName()
- + " does not have privilage to see this aplication " + appId);
+ + " does not have privilage to see this application " + appId);
}
return response;
}
@@ -490,7 +490,7 @@ public class ClientRMService extends AbstractService implements
.createContainerReport());
} else {
throw new YarnException("User " + callerUGI.getShortUserName()
- + " does not have privilage to see this aplication " + appId);
+ + " does not have privilage to see this application " + appId);
}
return response;
}
@@ -543,7 +543,7 @@ public class ClientRMService extends AbstractService implements
response = GetContainersResponse.newInstance(listContainers);
} else {
throw new YarnException("User " + callerUGI.getShortUserName()
- + " does not have privilage to see this aplication " + appId);
+ + " does not have privilage to see this application " + appId);
}
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f58f742/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 98dc87a..0f892cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1338,10 +1338,10 @@ public class TestClientRMService {
appPriority = 11;
ClientRMService rmService = rm.getClientRMService();
- testAplicationPriorityUpdation(rmService, app1, appPriority, maxPriority);
+ testApplicationPriorityUpdation(rmService, app1, appPriority, maxPriority);
appPriority = 9;
- testAplicationPriorityUpdation(rmService, app1, appPriority, appPriority);
+ testApplicationPriorityUpdation(rmService, app1, appPriority, appPriority);
rm.killApp(app1.getApplicationId());
rm.waitForState(app1.getApplicationId(), RMAppState.KILLED);
@@ -1370,7 +1370,7 @@ public class TestClientRMService {
rm.stop();
}
- private void testAplicationPriorityUpdation(ClientRMService rmService,
+ private void testApplicationPriorityUpdation(ClientRMService rmService,
RMApp app1, int tobeUpdatedPriority, int expected) throws YarnException,
IOException {
UpdateApplicationPriorityRequest updateRequest =
[36/50] [abbrv] hadoop git commit: HDFS-8898. Create API and
command-line argument to get quota and quota usage without detailed content
summary. Contributed by Ming Ma.
Posted by ka...@apache.org.
HDFS-8898. Create API and command-line argument to get quota and quota usage without detailed content summary. Contributed by Ming Ma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2fd19b96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2fd19b96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2fd19b96
Branch: refs/heads/YARN-1011
Commit: 2fd19b9674420e025af54a5bed12eb96478f8c48
Parents: d6258b3
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Jan 21 12:04:14 2016 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Jan 22 09:10:06 2016 -0600
----------------------------------------------------------------------
.../org/apache/hadoop/fs/ContentSummary.java | 241 ++++---------
.../java/org/apache/hadoop/fs/FileSystem.java | 7 +
.../java/org/apache/hadoop/fs/QuotaUsage.java | 359 +++++++++++++++++++
.../java/org/apache/hadoop/fs/shell/Count.java | 37 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 7 +-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 15 +-
.../src/site/markdown/FileSystemShell.md | 11 +-
.../org/apache/hadoop/cli/CLITestHelper.java | 2 +-
.../apache/hadoop/fs/TestFilterFileSystem.java | 3 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestQuotaUsage.java | 146 ++++++++
.../org/apache/hadoop/fs/shell/TestCount.java | 109 +++++-
.../src/test/resources/testConf.xml | 2 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 27 ++
.../hadoop/hdfs/DistributedFileSystem.java | 19 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 14 +
.../ClientNamenodeProtocolTranslatorPB.java | 14 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 79 +++-
.../src/main/proto/ClientNamenodeProtocol.proto | 10 +
.../src/main/proto/hdfs.proto | 11 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
...tNamenodeProtocolServerSideTranslatorPB.java | 16 +
.../server/namenode/FSDirStatAndListingOp.java | 53 +++
.../hdfs/server/namenode/FSNamesystem.java | 31 ++
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +
.../fs/viewfs/TestViewFsDefaultValue.java | 53 ++-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 66 +++-
.../server/namenode/ha/TestQuotasWithHA.java | 14 +-
28 files changed, 1125 insertions(+), 232 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index 678ce7f..3dedbcc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -30,26 +30,15 @@ import org.apache.hadoop.util.StringUtils;
/** Store the summary of a content (a directory or a file). */
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class ContentSummary implements Writable{
+public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
- private long quota;
- private long spaceConsumed;
- private long spaceQuota;
- private long typeConsumed[];
- private long typeQuota[];
- public static class Builder{
+ /** We don't use generics. Instead override spaceConsumed and other methods
+ in order to keep backward compatibility. */
+ public static class Builder extends QuotaUsage.Builder {
public Builder() {
- this.quota = -1;
- this.spaceQuota = -1;
-
- typeConsumed = new long[StorageType.values().length];
- typeQuota = new long[StorageType.values().length];
- for (int i = 0; i < typeQuota.length; i++) {
- typeQuota[i] = -1;
- }
}
public Builder length(long length) {
@@ -67,58 +56,57 @@ public class ContentSummary implements Writable{
return this;
}
+ @Override
public Builder quota(long quota){
- this.quota = quota;
+ super.quota(quota);
return this;
}
+ @Override
public Builder spaceConsumed(long spaceConsumed) {
- this.spaceConsumed = spaceConsumed;
+ super.spaceConsumed(spaceConsumed);
return this;
}
+ @Override
public Builder spaceQuota(long spaceQuota) {
- this.spaceQuota = spaceQuota;
+ super.spaceQuota(spaceQuota);
return this;
}
+ @Override
public Builder typeConsumed(long typeConsumed[]) {
- for (int i = 0; i < typeConsumed.length; i++) {
- this.typeConsumed[i] = typeConsumed[i];
- }
+ super.typeConsumed(typeConsumed);
return this;
}
+ @Override
public Builder typeQuota(StorageType type, long quota) {
- this.typeQuota[type.ordinal()] = quota;
+ super.typeQuota(type, quota);
return this;
}
+ @Override
public Builder typeConsumed(StorageType type, long consumed) {
- this.typeConsumed[type.ordinal()] = consumed;
+ super.typeConsumed(type, consumed);
return this;
}
+ @Override
public Builder typeQuota(long typeQuota[]) {
- for (int i = 0; i < typeQuota.length; i++) {
- this.typeQuota[i] = typeQuota[i];
- }
+ super.typeQuota(typeQuota);
return this;
}
public ContentSummary build() {
- return new ContentSummary(length, fileCount, directoryCount, quota,
- spaceConsumed, spaceQuota, typeConsumed, typeQuota);
+ // Set it in case applications call QuotaUsage#getFileAndDirectoryCount.
+ super.fileAndDirectoryCount(this.fileCount + this.directoryCount);
+ return new ContentSummary(this);
}
private long length;
private long fileCount;
private long directoryCount;
- private long quota;
- private long spaceConsumed;
- private long spaceQuota;
- private long typeConsumed[];
- private long typeQuota[];
}
/** Constructor deprecated by ContentSummary.Builder*/
@@ -143,24 +131,17 @@ public class ContentSummary implements Writable{
this.length = length;
this.fileCount = fileCount;
this.directoryCount = directoryCount;
- this.quota = quota;
- this.spaceConsumed = spaceConsumed;
- this.spaceQuota = spaceQuota;
+ setQuota(quota);
+ setSpaceConsumed(spaceConsumed);
+ setSpaceQuota(spaceQuota);
}
/** Constructor for ContentSummary.Builder*/
- private ContentSummary(
- long length, long fileCount, long directoryCount, long quota,
- long spaceConsumed, long spaceQuota, long typeConsumed[],
- long typeQuota[]) {
- this.length = length;
- this.fileCount = fileCount;
- this.directoryCount = directoryCount;
- this.quota = quota;
- this.spaceConsumed = spaceConsumed;
- this.spaceQuota = spaceQuota;
- this.typeConsumed = typeConsumed;
- this.typeQuota = typeQuota;
+ private ContentSummary(Builder builder) {
+ super(builder);
+ this.length = builder.length;
+ this.fileCount = builder.fileCount;
+ this.directoryCount = builder.directoryCount;
}
/** @return the length */
@@ -171,51 +152,6 @@ public class ContentSummary implements Writable{
/** @return the file count */
public long getFileCount() {return fileCount;}
-
- /** Return the directory quota */
- public long getQuota() {return quota;}
-
- /** Returns storage space consumed */
- public long getSpaceConsumed() {return spaceConsumed;}
-
- /** Returns storage space quota */
- public long getSpaceQuota() {return spaceQuota;}
-
- /** Returns storage type quota */
- public long getTypeQuota(StorageType type) {
- return (typeQuota != null) ? typeQuota[type.ordinal()] : -1;
- }
-
- /** Returns storage type consumed*/
- public long getTypeConsumed(StorageType type) {
- return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0;
- }
-
- /** Returns true if any storage type quota has been set*/
- public boolean isTypeQuotaSet() {
- if (typeQuota == null) {
- return false;
- }
- for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeQuota[t.ordinal()] > 0) {
- return true;
- }
- }
- return false;
- }
-
- /** Returns true if any storage type consumption information is available*/
- public boolean isTypeConsumedAvailable() {
- if (typeConsumed == null) {
- return false;
- }
- for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeConsumed[t.ordinal()] > 0) {
- return true;
- }
- }
- return false;
- }
@Override
@InterfaceAudience.Private
@@ -223,9 +159,9 @@ public class ContentSummary implements Writable{
out.writeLong(length);
out.writeLong(fileCount);
out.writeLong(directoryCount);
- out.writeLong(quota);
- out.writeLong(spaceConsumed);
- out.writeLong(spaceQuota);
+ out.writeLong(getQuota());
+ out.writeLong(getSpaceConsumed());
+ out.writeLong(getSpaceQuota());
}
@Override
@@ -234,9 +170,29 @@ public class ContentSummary implements Writable{
this.length = in.readLong();
this.fileCount = in.readLong();
this.directoryCount = in.readLong();
- this.quota = in.readLong();
- this.spaceConsumed = in.readLong();
- this.spaceQuota = in.readLong();
+ setQuota(in.readLong());
+ setSpaceConsumed(in.readLong());
+ setSpaceQuota(in.readLong());
+ }
+
+ @Override
+ public boolean equals(Object to) {
+ if (this == to) {
+ return true;
+ } else if (to instanceof ContentSummary) {
+ return getLength() == ((ContentSummary) to).getLength() &&
+ getFileCount() == ((ContentSummary) to).getFileCount() &&
+ getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() &&
+ super.equals(to);
+ } else {
+ return super.equals(to);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ long result = getLength() ^ getFileCount() ^ getDirectoryCount();
+ return ((int) result) ^ super.hashCode();
}
/**
@@ -245,35 +201,16 @@ public class ContentSummary implements Writable{
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
- /**
- * Output format:
- * <----12----> <------15-----> <------15-----> <------15----->
- * QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
- * <----12----> <----12----> <-------18------->
- * DIR_COUNT FILE_COUNT CONTENT_SIZE
- */
- private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
- private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";
-
- private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%13s %17s ";
- private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
- "FILE_COUNT", "CONTENT_SIZE"};
- private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
- "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };
+ private static final String[] SUMMARY_HEADER_FIELDS =
+ new String[] {"DIR_COUNT", "FILE_COUNT", "CONTENT_SIZE"};
/** The header string */
- private static final String HEADER = String.format(
- SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);
+ private static final String SUMMARY_HEADER = String.format(
+ SUMMARY_FORMAT, (Object[]) SUMMARY_HEADER_FIELDS);
- private static final String QUOTA_HEADER = String.format(
- QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
- (Object[]) QUOTA_HEADER_FIELDS) +
- HEADER;
+ private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
- /** default quota display string */
- private static final String QUOTA_NONE = "none";
- private static final String QUOTA_INF = "inf";
/** Return the header of the output.
* if qOption is false, output directory count, file count, and content size;
@@ -283,28 +220,10 @@ public class ContentSummary implements Writable{
* @return the header of the output
*/
public static String getHeader(boolean qOption) {
- return qOption ? QUOTA_HEADER : HEADER;
+ return qOption ? ALL_HEADER : SUMMARY_HEADER;
}
- /**
- * return the header of with the StorageTypes
- *
- * @param storageTypes
- * @return storage header string
- */
- public static String getStorageTypeHeader(List<StorageType> storageTypes) {
- StringBuffer header = new StringBuffer();
-
- for (StorageType st : storageTypes) {
- /* the field length is 13/17 for quota and remain quota
- * as the max length for quota name is ARCHIVE_QUOTA
- * and remain quota name REM_ARCHIVE_QUOTA */
- String storageName = st.toString();
- header.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT, storageName + "_QUOTA",
- "REM_" + storageName + "_QUOTA"));
- }
- return header.toString();
- }
+
/**
* Returns the names of the fields from the summary header.
@@ -312,7 +231,7 @@ public class ContentSummary implements Writable{
* @return names of fields as displayed in the header
*/
public static String[] getHeaderFields() {
- return HEADER_FIELDS;
+ return SUMMARY_HEADER_FIELDS;
}
/**
@@ -370,41 +289,11 @@ public class ContentSummary implements Writable{
String prefix = "";
if (tOption) {
- StringBuffer content = new StringBuffer();
- for (StorageType st : types) {
- long typeQuota = getTypeQuota(st);
- long typeConsumed = getTypeConsumed(st);
- String quotaStr = QUOTA_NONE;
- String quotaRem = QUOTA_INF;
-
- if (typeQuota > 0) {
- quotaStr = formatSize(typeQuota, hOption);
- quotaRem = formatSize(typeQuota - typeConsumed, hOption);
- }
-
- content.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT,
- quotaStr, quotaRem));
- }
- return content.toString();
+ return getTypesQuotaUsage(hOption, types);
}
if (qOption) {
- String quotaStr = QUOTA_NONE;
- String quotaRem = QUOTA_INF;
- String spaceQuotaStr = QUOTA_NONE;
- String spaceQuotaRem = QUOTA_INF;
-
- if (quota>0) {
- quotaStr = formatSize(quota, hOption);
- quotaRem = formatSize(quota-(directoryCount+fileCount), hOption);
- }
- if (spaceQuota>0) {
- spaceQuotaStr = formatSize(spaceQuota, hOption);
- spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
- }
-
- prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
- quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
+ prefix = getQuotaUsage(hOption);
}
return prefix + String.format(SUMMARY_FORMAT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index a3fbe33..920c5a6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1496,6 +1496,13 @@ public abstract class FileSystem extends Configured implements Closeable {
spaceConsumed(summary[0]).build();
}
+ /** Return the {@link QuotaUsage} of a given {@link Path}.
+ * @param f path to use
+ */
+ public QuotaUsage getQuotaUsage(Path f) throws IOException {
+ return getContentSummary(f);
+ }
+
final private static PathFilter DEFAULT_FILTER = new PathFilter() {
@Override
public boolean accept(Path file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
new file mode 100644
index 0000000..9ffab2f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -0,0 +1,359 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.StringUtils;
+
+/** Store the quota usage of a directory. */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class QuotaUsage {
+ private long fileAndDirectoryCount;
+ // Make the followings protected so that
+ // deprecated ContentSummary constructor can use them.
+ private long quota;
+ private long spaceConsumed;
+ private long spaceQuota;
+ private long[] typeConsumed;
+ private long[] typeQuota;
+
+ /** Builder class for QuotaUsage. */
+ public static class Builder {
+ public Builder() {
+ this.quota = -1;
+ this.spaceQuota = -1;
+
+ typeConsumed = new long[StorageType.values().length];
+ typeQuota = new long[StorageType.values().length];
+ for (int i = 0; i < typeQuota.length; i++) {
+ typeQuota[i] = -1;
+ }
+ }
+
+ public Builder fileAndDirectoryCount(long count) {
+ this.fileAndDirectoryCount = count;
+ return this;
+ }
+
+ public Builder quota(long quota){
+ this.quota = quota;
+ return this;
+ }
+
+ public Builder spaceConsumed(long spaceConsumed) {
+ this.spaceConsumed = spaceConsumed;
+ return this;
+ }
+
+ public Builder spaceQuota(long spaceQuota) {
+ this.spaceQuota = spaceQuota;
+ return this;
+ }
+
+ public Builder typeConsumed(long[] typeConsumed) {
+ for (int i = 0; i < typeConsumed.length; i++) {
+ this.typeConsumed[i] = typeConsumed[i];
+ }
+ return this;
+ }
+
+ public Builder typeQuota(StorageType type, long quota) {
+ this.typeQuota[type.ordinal()] = quota;
+ return this;
+ }
+
+ public Builder typeConsumed(StorageType type, long consumed) {
+ this.typeConsumed[type.ordinal()] = consumed;
+ return this;
+ }
+
+ public Builder typeQuota(long[] typeQuota) {
+ for (int i = 0; i < typeQuota.length; i++) {
+ this.typeQuota[i] = typeQuota[i];
+ }
+ return this;
+ }
+
+ public QuotaUsage build() {
+ return new QuotaUsage(this);
+ }
+
+ private long fileAndDirectoryCount;
+ private long quota;
+ private long spaceConsumed;
+ private long spaceQuota;
+ private long[] typeConsumed;
+ private long[] typeQuota;
+ }
+
+ // Make it protected for the deprecated ContentSummary constructor.
+ protected QuotaUsage() { }
+
+ /** Build the instance based on the builder. */
+ protected QuotaUsage(Builder builder) {
+ this.fileAndDirectoryCount = builder.fileAndDirectoryCount;
+ this.quota = builder.quota;
+ this.spaceConsumed = builder.spaceConsumed;
+ this.spaceQuota = builder.spaceQuota;
+ this.typeConsumed = builder.typeConsumed;
+ this.typeQuota = builder.typeQuota;
+ }
+
+ protected void setQuota(long quota) {
+ this.quota = quota;
+ }
+
+ protected void setSpaceConsumed(long spaceConsumed) {
+ this.spaceConsumed = spaceConsumed;
+ }
+
+ protected void setSpaceQuota(long spaceQuota) {
+ this.spaceQuota = spaceQuota;
+ }
+
+ /** Return the directory count. */
+ public long getFileAndDirectoryCount() {
+ return fileAndDirectoryCount;
+ }
+
+ /** Return the directory quota. */
+ public long getQuota() {
+ return quota;
+ }
+
+ /** Return (disk) space consumed. */
+ public long getSpaceConsumed() {
+ return spaceConsumed;
+ }
+
+ /** Return (disk) space quota. */
+ public long getSpaceQuota() {
+ return spaceQuota;
+ }
+
+ /** Return storage type quota. */
+ public long getTypeQuota(StorageType type) {
+ return (typeQuota != null) ? typeQuota[type.ordinal()] : -1;
+ }
+
+ /** Return storage type consumed. */
+ public long getTypeConsumed(StorageType type) {
+ return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0;
+ }
+
+ /** Return storage type quota. */
+ private long[] getTypesQuota() {
+ return typeQuota;
+ }
+
+ /** Return storage type quota. */
+ private long[] getTypesConsumed() {
+ return typeConsumed;
+ }
+
+ /** Return true if any storage type quota has been set. */
+ public boolean isTypeQuotaSet() {
+ if (typeQuota == null) {
+ return false;
+ }
+ for (StorageType t : StorageType.getTypesSupportingQuota()) {
+ if (typeQuota[t.ordinal()] > 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Return true if any storage type consumption information is available. */
+ public boolean isTypeConsumedAvailable() {
+ if (typeConsumed == null) {
+ return false;
+ }
+ for (StorageType t : StorageType.getTypesSupportingQuota()) {
+ if (typeConsumed[t.ordinal()] > 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object to) {
+ return (this == to || (to instanceof QuotaUsage &&
+ getFileAndDirectoryCount() ==
+ ((QuotaUsage) to).getFileAndDirectoryCount() &&
+ getQuota() == ((QuotaUsage) to).getQuota() &&
+ getSpaceConsumed() == ((QuotaUsage) to).getSpaceConsumed() &&
+ getSpaceQuota() == ((QuotaUsage) to).getSpaceQuota() &&
+ Arrays.equals(getTypesQuota(), ((QuotaUsage) to).getTypesQuota()) &&
+ Arrays.equals(getTypesConsumed(),
+ ((QuotaUsage) to).getTypesConsumed())));
+ }
+
+ @Override
+ public int hashCode() {
+ long result = (getFileAndDirectoryCount() ^ getQuota() ^
+ getSpaceConsumed() ^ getSpaceQuota());
+ if (getTypesQuota() != null) {
+ for (long quota : getTypesQuota()) {
+ result ^= quota;
+ }
+ }
+ if (getTypesConsumed() != null) {
+ for (long consumed : getTypesConsumed()) {
+ result ^= consumed;
+ }
+ }
+ return (int)result;
+ }
+
+ /**
+ * Output format:
+ * <----12----> <----15----> <----15----> <----15----> <-------18------->
+ * QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME
+ */
+ protected static final String QUOTA_STRING_FORMAT = "%12s %15s ";
+ protected static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
+
+ protected static final String[] QUOTA_HEADER_FIELDS = new String[] {"QUOTA",
+ "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA"};
+
+ protected static final String QUOTA_HEADER = String.format(
+ QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
+ (Object[]) QUOTA_HEADER_FIELDS);
+
+ /**
+ * Output format:
+ * <----12----> <------15-----> <------15-----> <------15----->
+ * QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
+ * <----12----> <----12----> <-------18------->
+ * DIR_COUNT FILE_COUNT CONTENT_SIZE
+ */
+ private static final String STORAGE_TYPE_SUMMARY_FORMAT = "%13s %17s ";
+
+ /** Return the header of the output.
+ * @return the header of the output
+ */
+ public static String getHeader() {
+ return QUOTA_HEADER;
+ }
+
+ /** default quota display string */
+ private static final String QUOTA_NONE = "none";
+ private static final String QUOTA_INF = "inf";
+
+ @Override
+ public String toString() {
+ return toString(false);
+ }
+
+ public String toString(boolean hOption) {
+ return toString(hOption, false, null);
+ }
+
+ /** Return the string representation of the object in the output format.
+ * if hOption is false file sizes are returned in bytes
+ * if hOption is true file sizes are returned in human readable
+ *
+ * @param hOption a flag indicating if human readable output if to be used
+ * @return the string representation of the object
+ */
+ public String toString(boolean hOption,
+ boolean tOption, List<StorageType> types) {
+ if (tOption) {
+ return getTypesQuotaUsage(hOption, types);
+ }
+ return getQuotaUsage(hOption);
+ }
+
+ protected String getQuotaUsage(boolean hOption) {
+ String quotaStr = QUOTA_NONE;
+ String quotaRem = QUOTA_INF;
+ String spaceQuotaStr = QUOTA_NONE;
+ String spaceQuotaRem = QUOTA_INF;
+
+ if (quota > 0) {
+ quotaStr = formatSize(quota, hOption);
+ quotaRem = formatSize(quota-fileAndDirectoryCount, hOption);
+ }
+ if (spaceQuota > 0) {
+ spaceQuotaStr = formatSize(spaceQuota, hOption);
+ spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
+ }
+
+ return String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
+ quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
+ }
+
+ protected String getTypesQuotaUsage(boolean hOption,
+ List<StorageType> types) {
+ StringBuffer content = new StringBuffer();
+ for (StorageType st : types) {
+ long typeQuota = getTypeQuota(st);
+ long typeConsumed = getTypeConsumed(st);
+ String quotaStr = QUOTA_NONE;
+ String quotaRem = QUOTA_INF;
+
+ if (typeQuota > 0) {
+ quotaStr = formatSize(typeQuota, hOption);
+ quotaRem = formatSize(typeQuota - typeConsumed, hOption);
+ }
+
+ content.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT,
+ quotaStr, quotaRem));
+ }
+ return content.toString();
+ }
+
+ /**
+ * return the header of with the StorageTypes.
+ *
+ * @param storageTypes
+ * @return storage header string
+ */
+ public static String getStorageTypeHeader(List<StorageType> storageTypes) {
+ StringBuffer header = new StringBuffer();
+
+ for (StorageType st : storageTypes) {
+ /* the field length is 13/17 for quota and remain quota
+ * as the max length for quota name is ARCHIVE_QUOTA
+ * and remain quota name REM_ARCHIVE_QUOTA */
+ String storageName = st.toString();
+ header.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT,
+ storageName + "_QUOTA", "REM_" + storageName + "_QUOTA"));
+ }
+ return header.toString();
+ }
+
+ /**
+ * Formats a size to be human readable or in bytes.
+ * @param size value to be formatted
+ * @param humanReadable flag indicating human readable or not
+ * @return String representation of the size
+ */
+ private String formatSize(long size, boolean humanReadable) {
+ return humanReadable
+ ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
+ : String.valueOf(size);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index c615876..2b5a4cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
/**
@@ -50,11 +51,14 @@ public class Count extends FsCommand {
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
private static final String OPTION_TYPE = "t";
+ //return the quota, namespace count and disk space usage.
+ private static final String OPTION_QUOTA_AND_USAGE = "u";
public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
- + "] [-" + OPTION_TYPE + " [<storage type>]] <path> ...";
+ + "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
+ OPTION_QUOTA_AND_USAGE + "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
@@ -74,12 +78,15 @@ public class Count extends FsCommand {
OPTION_TYPE + " option, \n" +
"it displays the quota and usage for the specified types. \n" +
"Otherwise, it displays the quota and usage for all the storage \n" +
- "types that support quota";
+ "types that support quota \n" +
+ "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
+ "the usage against the quota without the detailed content summary.";
private boolean showQuotas;
private boolean humanReadable;
private boolean showQuotabyType;
private List<StorageType> storageTypes = null;
+ private boolean showQuotasAndUsageOnly;
/** Constructor */
public Count() {}
@@ -99,7 +106,7 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
- OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
+ OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@@ -107,8 +114,9 @@ public class Count extends FsCommand {
}
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
+ showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
- if (showQuotas) {
+ if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
if (null != types) {
@@ -121,9 +129,13 @@ public class Count extends FsCommand {
if (cf.getOpt(OPTION_HEADER)) {
if (showQuotabyType) {
- out.println(ContentSummary.getStorageTypeHeader(storageTypes) + "PATHNAME");
+ out.println(QuotaUsage.getStorageTypeHeader(storageTypes) + "PATHNAME");
} else {
- out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
+ if (showQuotasAndUsageOnly) {
+ out.println(QuotaUsage.getHeader() + "PATHNAME");
+ } else {
+ out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
+ }
}
}
}
@@ -145,11 +157,16 @@ public class Count extends FsCommand {
@Override
protected void processPath(PathData src) throws IOException {
- ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas, isHumanReadable(),
- showQuotabyType, storageTypes) + src);
+ if (showQuotasAndUsageOnly || showQuotabyType) {
+ QuotaUsage usage = src.fs.getQuotaUsage(src.path);
+ out.println(usage.toString(isHumanReadable(), showQuotabyType,
+ storageTypes) + src);
+ } else {
+ ContentSummary summary = src.fs.getContentSummary(src.path);
+ out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+ }
}
-
+
/**
* Should quotas get shown as part of the report?
* @return if quotas should be shown then true otherwise false
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index dd28c58..46e8f2d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
@@ -388,7 +389,11 @@ class ChRootedFileSystem extends FilterFileSystem {
public ContentSummary getContentSummary(Path f) throws IOException {
return fs.getContentSummary(fullPath(f));
}
-
+
+ @Override
+ public QuotaUsage getQuotaUsage(Path f) throws IOException {
+ return fs.getQuotaUsage(fullPath(f));
+ }
private static Path rootPath = new Path(Path.SEPARATOR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 8605064..5f42e70 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
import org.apache.hadoop.security.AccessControlException;
@@ -689,12 +690,19 @@ public class ViewFileSystem extends FileSystem {
@Override
public ContentSummary getContentSummary(Path f) throws IOException {
- InodeTree.ResolveResult<FileSystem> res =
+ InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(f), true);
return res.targetFileSystem.getContentSummary(res.remainingPath);
}
@Override
+ public QuotaUsage getQuotaUsage(Path f) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(f), true);
+ return res.targetFileSystem.getQuotaUsage(res.remainingPath);
+ }
+
+ @Override
public void setWriteChecksum(final boolean writeChecksum) {
List<InodeTree.MountPoint<FileSystem>> mountPoints =
fsState.getMountPoints();
@@ -1066,5 +1074,10 @@ public class ViewFileSystem extends FileSystem {
checkPathIsSlash(path);
throw readOnlyMountTable("deleteSnapshot", path);
}
+
+ @Override
+ public QuotaUsage getQuotaUsage(Path f) throws IOException {
+ throw new NotInMountpointException(f, "getQuotaUsage");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index db8b785..ab66156 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -170,12 +170,16 @@ Similar to get command, except that the destination is restricted to a local fil
count
-----
-Usage: `hadoop fs -count [-q] [-h] [-v] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> `
-Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
+
+The -t option shows the quota and usage for each storage type.
+
The -h option shows sizes in human readable format.
The -v option displays a header line.
@@ -186,6 +190,9 @@ Example:
* `hadoop fs -count -q hdfs://nn1.example.com/file1`
* `hadoop fs -count -q -h hdfs://nn1.example.com/file1`
* `hadoop fs -count -q -h -v hdfs://nn1.example.com/file1`
+* `hadoop fs -count -u hdfs://nn1.example.com/file1`
+* `hadoop fs -count -u -h hdfs://nn1.example.com/file1`
+* `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1`
Exit Code:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
index d39727b..b08af16 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
@@ -68,7 +68,7 @@ public class CLITestHelper {
protected String clitestDataDir = null;
protected String username = null;
/**
- * Read the test config file - testConfig.xml
+ * Read the test config file - testConf.xml
*/
protected void readTestConfigFile() {
String testConfigFile = getTestFile();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
index abb75f5..3ecce8f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
@@ -131,8 +131,9 @@ public class TestFilterFileSystem {
public String getScheme();
public Path fixRelativePart(Path p);
public ContentSummary getContentSummary(Path f);
+ public QuotaUsage getQuotaUsage(Path f);
}
-
+
@Test
public void testFilterFileSystem() throws Exception {
int errors = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 8ec9d80..26923a8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -112,6 +112,7 @@ public class TestHarFileSystem {
public short getReplication(Path src);
public void processDeleteOnExit();
public ContentSummary getContentSummary(Path f);
+ public QuotaUsage getQuotaUsage(Path f);
public FsStatus getStatus();
public FileStatus[] listStatus(Path f, PathFilter filter);
public FileStatus[] listStatus(Path[] files);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestQuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestQuotaUsage.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestQuotaUsage.java
new file mode 100644
index 0000000..e3e2002
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestQuotaUsage.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestQuotaUsage {
+
+ // check the empty constructor correctly initialises the object
+ @Test
+ public void testConstructorEmpty() {
+ QuotaUsage quotaUsage = new QuotaUsage.Builder().build();
+ assertEquals("getQuota", -1, quotaUsage.getQuota());
+ assertEquals("getSpaceConsumed", 0, quotaUsage.getSpaceConsumed());
+ assertEquals("getSpaceQuota", -1, quotaUsage.getSpaceQuota());
+ }
+
+ // check the full constructor with quota information
+ @Test
+ public void testConstructorWithQuota() {
+ long fileAndDirCount = 22222;
+ long quota = 44444;
+ long spaceConsumed = 55555;
+ long spaceQuota = 66666;
+
+ QuotaUsage quotaUsage = new QuotaUsage.Builder().
+ fileAndDirectoryCount(fileAndDirCount).quota(quota).
+ spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).build();
+ assertEquals("getFileAndDirectoryCount", fileAndDirCount,
+ quotaUsage.getFileAndDirectoryCount());
+ assertEquals("getQuota", quota, quotaUsage.getQuota());
+ assertEquals("getSpaceConsumed", spaceConsumed,
+ quotaUsage.getSpaceConsumed());
+ assertEquals("getSpaceQuota", spaceQuota, quotaUsage.getSpaceQuota());
+ }
+
+ // check the constructor with quota information
+ @Test
+ public void testConstructorNoQuota() {
+ long spaceConsumed = 11111;
+ long fileAndDirCount = 22222;
+ QuotaUsage quotaUsage = new QuotaUsage.Builder().
+ fileAndDirectoryCount(fileAndDirCount).
+ spaceConsumed(spaceConsumed).build();
+ assertEquals("getFileAndDirectoryCount", fileAndDirCount,
+ quotaUsage.getFileAndDirectoryCount());
+ assertEquals("getQuota", -1, quotaUsage.getQuota());
+ assertEquals("getSpaceConsumed", spaceConsumed,
+ quotaUsage.getSpaceConsumed());
+ assertEquals("getSpaceQuota", -1, quotaUsage.getSpaceQuota());
+ }
+
+ // check the header
+ @Test
+ public void testGetHeader() {
+ String header = " QUOTA REM_QUOTA SPACE_QUOTA "
+ + "REM_SPACE_QUOTA ";
+ assertEquals(header, QuotaUsage.getHeader());
+ }
+
+ // check the toString method with quotas
+ @Test
+ public void testToStringWithQuota() {
+ long fileAndDirCount = 55555;
+ long quota = 44444;
+ long spaceConsumed = 55555;
+ long spaceQuota = 66665;
+
+ QuotaUsage quotaUsage = new QuotaUsage.Builder().
+ fileAndDirectoryCount(fileAndDirCount).quota(quota).
+ spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).build();
+ String expected =" 44444 -11111 66665" +
+ " 11110 ";
+ assertEquals(expected, quotaUsage.toString());
+ }
+
+ // check the toString method with quotas
+ @Test
+ public void testToStringNoQuota() {
+ QuotaUsage quotaUsage = new QuotaUsage.Builder().
+ fileAndDirectoryCount(1234).build();
+ String expected = " none inf none"
+ + " inf ";
+ assertEquals(expected, quotaUsage.toString());
+ }
+
+ // check the toString method with quotas
+ @Test
+ public void testToStringHumanWithQuota() {
+ long fileAndDirCount = 222255555;
+ long quota = 222256578;
+ long spaceConsumed = 1073741825;
+ long spaceQuota = 1;
+
+ QuotaUsage quotaUsage = new QuotaUsage.Builder().
+ fileAndDirectoryCount(fileAndDirCount).quota(quota).
+ spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).build();
+ String expected = " 212.0 M 1023 1 "
+ + " -1 G ";
+ assertEquals(expected, quotaUsage.toString(true));
+ }
+
+ // check the equality
+ @Test
+ public void testCompareQuotaUsage() {
+ long fileAndDirCount = 222255555;
+ long quota = 222256578;
+ long spaceConsumed = 1073741825;
+ long spaceQuota = 1;
+ long SSDspaceConsumed = 100000;
+ long SSDQuota = 300000;
+
+ QuotaUsage quotaUsage1 = new QuotaUsage.Builder().
+ fileAndDirectoryCount(fileAndDirCount).quota(quota).
+ spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).
+ typeConsumed(StorageType.SSD, SSDQuota).
+ typeQuota(StorageType.SSD, SSDQuota).
+ build();
+
+ QuotaUsage quotaUsage2 = new QuotaUsage.Builder().
+ fileAndDirectoryCount(fileAndDirCount).quota(quota).
+ spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).
+ typeConsumed(StorageType.SSD, SSDQuota).
+ typeQuota(StorageType.SSD, SSDQuota).
+ build();
+
+ assertEquals(quotaUsage1, quotaUsage2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index 116c071..04174e0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FilterFileSystem;
@@ -47,6 +48,7 @@ public class TestCount {
private static final String NO_QUOTAS = "Content summary without quotas";
private static final String HUMAN = "human: ";
private static final String BYTES = "bytes: ";
+ private static final String QUOTAS_AND_USAGE = "quotas and usage";
private static Configuration conf;
private static FileSystem mockFs;
private static FileStatus fileStat;
@@ -344,7 +346,20 @@ public class TestCount {
}
@Test
- public void processPathWithQuotasByMultipleStorageTypesContent() throws Exception {
+ public void processPathWithQuotasByMultipleStorageTypesContent()
+ throws Exception {
+ processMultipleStorageTypesContent(false);
+ }
+
+ @Test
+ public void processPathWithQuotaUsageByMultipleStorageTypesContent()
+ throws Exception {
+ processMultipleStorageTypesContent(true);
+ }
+
+ // "-q -t" is the same as "-u -t"; only return the storage quota and usage.
+ private void processMultipleStorageTypesContent(boolean quotaUsageOnly)
+ throws Exception {
Path path = new Path("mockfs:/test");
when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
@@ -356,7 +371,7 @@ public class TestCount {
count.out = out;
LinkedList<String> options = new LinkedList<String>();
- options.add("-q");
+ options.add(quotaUsageOnly ? "-u" : "-q");
options.add("-t");
options.add("SSD,DISK");
options.add("dummy");
@@ -431,7 +446,8 @@ public class TestCount {
public void getUsage() {
Count count = new Count();
String actual = count.getUsage();
- String expected = "-count [-q] [-h] [-v] [-t [<storage type>]] <path> ...";
+ String expected =
+ "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@@ -454,11 +470,47 @@ public class TestCount {
+ "If a comma-separated list of storage types is given after the -t option, \n"
+ "it displays the quota and usage for the specified types. \n"
+ "Otherwise, it displays the quota and usage for all the storage \n"
- + "types that support quota";
+ + "types that support quota \n"
+ + "The -u option shows the quota and \n"
+ + "the usage against the quota without the detailed content summary.";
assertEquals("Count.getDescription", expected, actual);
}
+ @Test
+ public void processPathWithQuotaUsageHuman() throws Exception {
+ processPathWithQuotaUsage(false);
+ }
+
+ @Test
+ public void processPathWithQuotaUsageRawBytes() throws Exception {
+ processPathWithQuotaUsage(true);
+ }
+
+ private void processPathWithQuotaUsage(boolean rawBytes) throws Exception {
+ Path path = new Path("mockfs:/test");
+
+ when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
+ PathData pathData = new PathData(path.toString(), conf);
+
+ PrintStream out = mock(PrintStream.class);
+
+ Count count = new Count();
+ count.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ if (!rawBytes) {
+ options.add("-h");
+ }
+ options.add("-u");
+ options.add("dummy");
+ count.processOptions(options);
+ count.processPath(pathData);
+ String withStorageType = (rawBytes ? BYTES : HUMAN) + QUOTAS_AND_USAGE +
+ pathData.toString();
+ verify(out).println(withStorageType);
+ verifyNoMoreInteractions(out);
+ }
// mock content system
static class MockContentSummary extends ContentSummary {
@@ -469,19 +521,7 @@ public class TestCount {
}
@Override
- public String toString(boolean qOption, boolean hOption,
- boolean tOption, List<StorageType> types) {
- if (tOption) {
- StringBuffer result = new StringBuffer();
- result.append(hOption ? HUMAN : BYTES);
-
- for (StorageType type : types) {
- result.append(type.toString());
- result.append(" ");
- }
- return result.toString();
- }
-
+ public String toString(boolean qOption, boolean hOption) {
if (qOption) {
if (hOption) {
return (HUMAN + WITH_QUOTAS);
@@ -498,6 +538,36 @@ public class TestCount {
}
}
+ // mock content system
+ static class MockQuotaUsage extends QuotaUsage {
+
+ @SuppressWarnings("deprecation")
+ // suppress warning on the usage of deprecated ContentSummary constructor
+ public MockQuotaUsage() {
+ }
+
+ @Override
+ public String toString(boolean hOption,
+ boolean tOption, List<StorageType> types) {
+ if (tOption) {
+ StringBuffer result = new StringBuffer();
+ result.append(hOption ? HUMAN : BYTES);
+
+ for (StorageType type : types) {
+ result.append(type.toString());
+ result.append(" ");
+ }
+ return result.toString();
+ }
+
+ if (hOption) {
+ return (HUMAN + QUOTAS_AND_USAGE);
+ } else {
+ return (BYTES + QUOTAS_AND_USAGE);
+ }
+ }
+ }
+
// mock file system for use in testing
static class MockFileSystem extends FilterFileSystem {
Configuration conf;
@@ -525,5 +595,10 @@ public class TestCount {
public Configuration getConf() {
return conf;
}
+
+ @Override
+ public QuotaUsage getQuotaUsage(Path f) throws IOException {
+ return new MockQuotaUsage();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 79ab282..3ad6d66 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -270,7 +270,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] <path> \.\.\. :( )*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] <path> \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 3b9210a..277990f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -87,6 +87,7 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.XAttr;
@@ -163,6 +164,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.RpcNoSuchMethodException;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -2442,6 +2444,31 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
+ * Get {@link QuotaUsage} rooted at the specified directory.
+ * @param src The string representation of the path
+ *
+ * @see ClientProtocol#getQuotaUsage(String)
+ */
+ QuotaUsage getQuotaUsage(String src) throws IOException {
+ checkOpen();
+ try (TraceScope ignored = newPathTraceScope("getQuotaUsage", src)) {
+ return namenode.getQuotaUsage(src);
+ } catch(RemoteException re) {
+ IOException ioe = re.unwrapRemoteException(AccessControlException.class,
+ FileNotFoundException.class,
+ UnresolvedPathException.class,
+ RpcNoSuchMethodException.class);
+ if (ioe instanceof RpcNoSuchMethodException) {
+ LOG.debug("The version of namenode doesn't support getQuotaUsage API." +
+ " Fall back to use getContentSummary API.");
+ return getContentSummary(src);
+ } else {
+ throw ioe;
+ }
+ }
+ }
+
+ /**
* Sets or resets quotas for a directory.
* @see ClientProtocol#setQuota(String, long, long, StorageType)
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 48efb1f..551e2e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -723,6 +724,24 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @Override
+ public QuotaUsage getQuotaUsage(Path f) throws IOException {
+ statistics.incrementReadOps(1);
+ Path absF = fixRelativePart(f);
+ return new FileSystemLinkResolver<QuotaUsage>() {
+ @Override
+ public QuotaUsage doCall(final Path p)
+ throws IOException, UnresolvedLinkException {
+ return dfs.getQuotaUsage(getPathName(p));
+ }
+ @Override
+ public QuotaUsage next(final FileSystem fs, final Path p)
+ throws IOException {
+ return fs.getQuotaUsage(p);
+ }
+ }.resolve(this, absF);
+ }
+
/** Set a directory's quotas
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String,
* long, long, StorageType)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 6ebb01d..e6a99fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -1514,4 +1515,17 @@ public interface ClientProtocol {
*/
@Idempotent
ErasureCodingPolicy getErasureCodingPolicy(String src) throws IOException;
+
+ /**
+ * Get {@link QuotaUsage} rooted at the specified directory.
+ * @param path The string representation of the path
+ *
+ * @throws AccessControlException permission denied
+ * @throws java.io.FileNotFoundException file <code>path</code> is not found
+ * @throws org.apache.hadoop.fs.UnresolvedLinkException if <code>path</code>
+ * contains a symlink.
+ * @throws IOException If an I/O error occurred
+ */
+ @Idempotent
+ QuotaUsage getQuotaUsage(String path) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index ff13b22..d9e6472 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -110,6 +111,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLin
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportResponseProto;
@@ -1544,4 +1546,16 @@ public class ClientNamenodeProtocolTranslatorPB implements
throw ProtobufHelper.getRemoteException(e);
}
}
+
+ @Override
+ public QuotaUsage getQuotaUsage(String path) throws IOException {
+ GetQuotaUsageRequestProto req =
+ GetQuotaUsageRequestProto.newBuilder().setPath(path).build();
+ try {
+ return PBHelperClient.convert(rpcProxy.getQuotaUsage(null, req)
+ .getUsage());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 934011b..1e48a69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -139,6 +140,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.File
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RollingUpgradeStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto;
@@ -1417,12 +1419,37 @@ public class PBHelperClient {
spaceConsumed(cs.getSpaceConsumed()).
spaceQuota(cs.getSpaceQuota());
if (cs.hasTypeQuotaInfos()) {
- for (HdfsProtos.StorageTypeQuotaInfoProto info :
- cs.getTypeQuotaInfos().getTypeQuotaInfoList()) {
- StorageType type = convertStorageType(info.getType());
- builder.typeConsumed(type, info.getConsumed());
- builder.typeQuota(type, info.getQuota());
- }
+ addStorageTypes(cs.getTypeQuotaInfos(), builder);
+ }
+ return builder.build();
+ }
+
+ public static QuotaUsage convert(QuotaUsageProto qu) {
+ if (qu == null) {
+ return null;
+ }
+ QuotaUsage.Builder builder = new QuotaUsage.Builder();
+ builder.fileAndDirectoryCount(qu.getFileAndDirectoryCount()).
+ quota(qu.getQuota()).
+ spaceConsumed(qu.getSpaceConsumed()).
+ spaceQuota(qu.getSpaceQuota());
+ if (qu.hasTypeQuotaInfos()) {
+ addStorageTypes(qu.getTypeQuotaInfos(), builder);
+ }
+ return builder.build();
+ }
+
+ public static QuotaUsageProto convert(QuotaUsage qu) {
+ if (qu == null) {
+ return null;
+ }
+ QuotaUsageProto.Builder builder = QuotaUsageProto.newBuilder();
+ builder.setFileAndDirectoryCount(qu.getFileAndDirectoryCount()).
+ setQuota(qu.getQuota()).
+ setSpaceConsumed(qu.getSpaceConsumed()).
+ setSpaceQuota(qu.getSpaceQuota());
+ if (qu.isTypeQuotaSet() || qu.isTypeConsumedAvailable()) {
+ builder.setTypeQuotaInfos(getBuilder(qu));
}
return builder.build();
}
@@ -1989,22 +2016,38 @@ public class PBHelperClient {
setSpaceQuota(cs.getSpaceQuota());
if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
- HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
- HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
- for (StorageType t: StorageType.getTypesSupportingQuota()) {
- HdfsProtos.StorageTypeQuotaInfoProto info =
- HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
- setType(convertStorageType(t)).
- setConsumed(cs.getTypeConsumed(t)).
- setQuota(cs.getTypeQuota(t)).
- build();
- isb.addTypeQuotaInfo(info);
- }
- builder.setTypeQuotaInfos(isb);
+ builder.setTypeQuotaInfos(getBuilder(cs));
}
return builder.build();
}
+ private static void addStorageTypes(
+ HdfsProtos.StorageTypeQuotaInfosProto typeQuotaInfos,
+ QuotaUsage.Builder builder) {
+ for (HdfsProtos.StorageTypeQuotaInfoProto info :
+ typeQuotaInfos.getTypeQuotaInfoList()) {
+ StorageType type = convertStorageType(info.getType());
+ builder.typeConsumed(type, info.getConsumed());
+ builder.typeQuota(type, info.getQuota());
+ }
+ }
+
+ private static HdfsProtos.StorageTypeQuotaInfosProto.Builder getBuilder(
+ QuotaUsage qu) {
+ HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
+ HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
+ for (StorageType t: StorageType.getTypesSupportingQuota()) {
+ HdfsProtos.StorageTypeQuotaInfoProto info =
+ HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
+ setType(convertStorageType(t)).
+ setConsumed(qu.getTypeConsumed(t)).
+ setQuota(qu.getTypeQuota(t)).
+ build();
+ isb.addTypeQuotaInfo(info);
+ }
+ return isb;
+ }
+
public static DatanodeStorageProto convert(DatanodeStorage s) {
return DatanodeStorageProto.newBuilder()
.setState(convertState(s.getState()))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 865f946..6a1f777 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -582,6 +582,14 @@ message GetContentSummaryResponseProto {
required ContentSummaryProto summary = 1;
}
+message GetQuotaUsageRequestProto {
+ required string path = 1;
+}
+
+message GetQuotaUsageResponseProto {
+ required QuotaUsageProto usage = 1;
+}
+
message SetQuotaRequestProto {
required string path = 1;
required uint64 namespaceQuota = 2;
@@ -879,4 +887,6 @@ service ClientNamenodeProtocol {
returns(GetErasureCodingPoliciesResponseProto);
rpc getErasureCodingPolicy(GetErasureCodingPolicyRequestProto)
returns(GetErasureCodingPolicyResponseProto);
+ rpc getQuotaUsage(GetQuotaUsageRequestProto)
+ returns(GetQuotaUsageResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index bc8f8dc..b6f91e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -139,6 +139,17 @@ message ContentSummaryProto {
}
/**
+ * Summary of quota usage of a directory
+ */
+message QuotaUsageProto {
+ required uint64 fileAndDirectoryCount = 1;
+ required uint64 quota = 2;
+ required uint64 spaceConsumed = 3;
+ required uint64 spaceQuota = 4;
+ optional StorageTypeQuotaInfosProto typeQuotaInfos = 5;
+}
+
+/**
* Storage type quota and usage information of a file or directory
*/
message StorageTypeQuotaInfosProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c7061e7..43adf62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1836,6 +1836,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9655. NN should start JVM pause monitor before loading fsimage.
(John Zhuge via Lei (Eddy) Xu)
+ HDFS-8898. Create API and command-line argument to get quota and quota
+ usage without detailed content summary. (Ming Ma via kihwal)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 1e4b899..13c0b14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -120,6 +121,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLis
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportRequestProto;
@@ -1567,4 +1570,17 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throw new ServiceException(e);
}
}
+
+ @Override
+ public GetQuotaUsageResponseProto getQuotaUsage(
+ RpcController controller, GetQuotaUsageRequestProto req)
+ throws ServiceException {
+ try {
+ QuotaUsage result = server.getQuotaUsage(req.getPath());
+ return GetQuotaUsageResponseProto.newBuilder()
+ .setUsage(PBHelperClient.convert(result)).build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 0aa073b..3acfcc9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -626,6 +627,58 @@ class FSDirStatAndListingOp {
}
}
+ static QuotaUsage getQuotaUsage(
+ FSDirectory fsd, String src) throws IOException {
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ final INodesInPath iip;
+ fsd.readLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ iip = fsd.getINodesInPath(src, false);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPermission(pc, iip, false, null, null, null,
+ FsAction.READ_EXECUTE);
+ }
+ } finally {
+ fsd.readUnlock();
+ }
+ QuotaUsage usage = getQuotaUsageInt(fsd, iip);
+ if (usage != null) {
+ return usage;
+ } else {
+ //If quota isn't set, fall back to getContentSummary.
+ return getContentSummaryInt(fsd, iip);
+ }
+ }
+
+ private static QuotaUsage getQuotaUsageInt(FSDirectory fsd, INodesInPath iip)
+ throws IOException {
+ fsd.readLock();
+ try {
+ INode targetNode = iip.getLastINode();
+ QuotaUsage usage = null;
+ if (targetNode.isDirectory()) {
+ DirectoryWithQuotaFeature feature =
+ targetNode.asDirectory().getDirectoryWithQuotaFeature();
+ if (feature != null) {
+ QuotaCounts counts = feature.getSpaceConsumed();
+ QuotaCounts quotas = feature.getQuota();
+ usage = new QuotaUsage.Builder().
+ fileAndDirectoryCount(counts.getNameSpace()).
+ quota(quotas.getNameSpace()).
+ spaceConsumed(counts.getStorageSpace()).
+ spaceQuota(quotas.getStorageSpace()).
+ typeConsumed(counts.getTypeSpaces().asArray()).
+ typeQuota(quotas.getTypeSpaces().asArray()).build();
+ }
+ }
+ return usage;
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
static class GetBlockLocationsResult {
final boolean updateAccessTime;
final LocatedBlocks blocks;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3d26544..abc0614 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -153,6 +153,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -2854,6 +2855,36 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * Get the quota usage for a specific file/dir.
+ *
+ * @param src The string representation of the path to the file
+ *
+ * @throws AccessControlException if access is denied
+ * @throws UnresolvedLinkException if a symlink is encountered.
+ * @throws FileNotFoundException if no file exists
+ * @throws StandbyException
+ * @throws IOException for issues with writing to the audit log
+ *
+ * @return object containing information regarding the file
+ * or null if file not found
+ */
+ QuotaUsage getQuotaUsage(final String src) throws IOException {
+ checkOperation(OperationCategory.READ);
+ readLock();
+ boolean success = true;
+ try {
+ checkOperation(OperationCategory.READ);
+ return FSDirStatAndListingOp.getQuotaUsage(dir, src);
+ } catch (AccessControlException ace) {
+ success = false;
+ throw ace;
+ } finally {
+ readUnlock();
+ logAuditEvent(success, "quotaUsage", src);
+ }
+ }
+
+ /**
* Set the namespace quota and storage space quota for a directory.
* See {@link ClientProtocol#setQuota(String, long, long, StorageType)} for the
* contract.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 60366ae..7785260 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.ha.ServiceFailedException;
@@ -1266,6 +1267,12 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
+ public QuotaUsage getQuotaUsage(String path) throws IOException {
+ checkNNStartup();
+ return namesystem.getQuotaUsage(path);
+ }
+
+ @Override // ClientProtocol
public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
StorageType type)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2fd19b96/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
index bed128b..8d7d2da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
@@ -29,6 +29,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRIT
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -44,6 +45,8 @@ import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.QuotaUsage;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.AfterClass;
@@ -160,7 +163,55 @@ public class TestViewFsDefaultValue {
assertEquals(100, cs.getQuota());
assertEquals(500, cs.getSpaceQuota());
}
-
+
+ /**
+ * Test that getQuotaUsage can be retrieved on the client side.
+ */
+ @Test
+ public void testGetQuotaUsage() throws IOException {
+ FileSystem hFs = cluster.getFileSystem(0);
+ final DistributedFileSystem dfs = (DistributedFileSystem)hFs;
+ dfs.setQuota(testFileDirPath, 100, 500);
+ QuotaUsage qu = vfs.getQuotaUsage(testFileDirPath);
+ assertEquals(100, qu.getQuota());
+ assertEquals(500, qu.getSpaceQuota());
+ }
+
+ /**
+ * Test that getQuotaUsage can be retrieved on the client side if
+ * storage types are defined.
+ */
+ @Test
+ public void testGetQuotaUsageWithStorageTypes() throws IOException {
+ FileSystem hFs = cluster.getFileSystem(0);
+ final DistributedFileSystem dfs = (DistributedFileSystem)hFs;
+ dfs.setQuotaByStorageType(testFileDirPath, StorageType.SSD, 500);
+ dfs.setQuotaByStorageType(testFileDirPath, StorageType.DISK, 600);
+ QuotaUsage qu = vfs.getQuotaUsage(testFileDirPath);
+ assertEquals(500, qu.getTypeQuota(StorageType.SSD));
+ assertEquals(600, qu.getTypeQuota(StorageType.DISK));
+ }
+
+ /**
+ * Test that getQuotaUsage can be retrieved on the client side if
+ * quota isn't defined.
+ */
+ @Test
+ public void testGetQuotaUsageWithQuotaDefined() throws IOException {
+ FileSystem hFs = cluster.getFileSystem(0);
+ final DistributedFileSystem dfs = (DistributedFileSystem)hFs;
+ dfs.setQuota(testFileDirPath, -1, -1);
+ dfs.setQuotaByStorageType(testFileDirPath, StorageType.SSD, -1);
+ dfs.setQuotaByStorageType(testFileDirPath, StorageType.DISK, -1);
+ QuotaUsage qu = vfs.getQuotaUsage(testFileDirPath);
+ assertEquals(-1, qu.getTypeQuota(StorageType.SSD));
+ assertEquals(-1, qu.getQuota());
+ assertEquals(-1, qu.getSpaceQuota());
+ assertEquals(2, qu.getFileAndDirectoryCount());
+ assertEquals(0, qu.getTypeConsumed(StorageType.SSD));
+ assertTrue(qu.getSpaceConsumed() > 0);
+ }
+
@AfterClass
public static void cleanup() throws IOException {
fHdfs.delete(new Path(testFileName), true);
[43/50] [abbrv] hadoop git commit: YARN-4614. Fix random failure in
TestApplicationPriority#testApplicationPriorityAllocationWithChangeInPriority.
(Sunil G via rohithsharmaks)
Posted by ka...@apache.org.
YARN-4614. Fix random failure in TestApplicationPriority#testApplicationPriorityAllocationWithChangeInPriority. (Sunil G via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99829eb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99829eb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99829eb2
Branch: refs/heads/YARN-1011
Commit: 99829eb221482928d8a1b148ae3c802cc7c9253e
Parents: 618bfd6
Author: rohithsharmaks <ro...@apache.org>
Authored: Sat Jan 23 07:56:15 2016 +0530
Committer: rohithsharmaks <ro...@apache.org>
Committed: Sat Jan 23 07:56:57 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../scheduler/capacity/TestApplicationPriority.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99829eb2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8e87f4a..1e9f83c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1319,6 +1319,9 @@ Release 2.8.0 - UNRELEASED
YARN-4605. Spelling mistake in the help message of "yarn applicationattempt"
command. (Weiwei Yang via aajisaka)
+ YARN-4614. Fix random failure in TestApplicationPriority#testApplicationPriority
+ AllocationWithChangeInPriority. (Sunil G via rohithsharmaks)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99829eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index e32a33b..1569a12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -490,7 +490,6 @@ public class TestApplicationPriority {
RMApp app2 = rm.submitApp(1 * GB, appPriority2);
// kick the scheduler, 1 GB which was free is given to AM of App2
- nm1.nodeHeartbeat(true);
MockAM am2 = MockRM.launchAM(app2, rm, nm1);
am2.registerAppAttempt();
[45/50] [abbrv] hadoop git commit: HDFS-9653. Added blocks pending
deletion report to dfsadmin. (Weiwei Yang via eyang)
Posted by ka...@apache.org.
HDFS-9653. Added blocks pending deletion report to dfsadmin.
(Weiwei Yang via eyang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10a2bc0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10a2bc0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10a2bc0d
Branch: refs/heads/YARN-1011
Commit: 10a2bc0dffaece216eb9a6bac3236a086b9ece31
Parents: 10dc2c0
Author: Eric Yang <ey...@apache.org>
Authored: Sun Jan 24 14:19:49 2016 -0800
Committer: Eric Yang <ey...@apache.org>
Committed: Sun Jan 24 14:19:49 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 34 ++++++++++++++------
.../hadoop/hdfs/DistributedFileSystem.java | 9 ++++++
.../hadoop/hdfs/protocol/ClientProtocol.java | 4 ++-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 7 ++++
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../blockmanagement/HeartbeatManager.java | 1 +
.../hdfs/server/namenode/FSNamesystem.java | 2 ++
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 ++
.../TestPendingInvalidateBlock.java | 19 +++++++++++
10 files changed, 71 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 277990f..02ef47e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1957,10 +1957,11 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
}
- private long[] callGetStats() throws IOException {
+ private long getStateByIndex(int stateIndex) throws IOException {
checkOpen();
try (TraceScope ignored = tracer.newScope("getStats")) {
- return namenode.getStats();
+ long[] states = namenode.getStats();
+ return states.length > stateIndex ? states[stateIndex] : -1;
}
}
@@ -1968,8 +1969,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @see ClientProtocol#getStats()
*/
public FsStatus getDiskStatus() throws IOException {
- long rawNums[] = callGetStats();
- return new FsStatus(rawNums[0], rawNums[1], rawNums[2]);
+ return new FsStatus(getStateByIndex(0),
+ getStateByIndex(1), getStateByIndex(2));
}
/**
@@ -1978,7 +1979,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @throws IOException
*/
public long getMissingBlocksCount() throws IOException {
- return callGetStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX];
+ return getStateByIndex(ClientProtocol.
+ GET_STATS_MISSING_BLOCKS_IDX);
}
/**
@@ -1987,8 +1989,17 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @throws IOException
*/
public long getMissingReplOneBlocksCount() throws IOException {
- return callGetStats()[ClientProtocol.
- GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX];
+ return getStateByIndex(ClientProtocol.
+ GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX);
+ }
+
+ /**
+ * Returns count of blocks pending on deletion.
+ * @throws IOException
+ */
+ public long getPendingDeletionBlocksCount() throws IOException {
+ return getStateByIndex(ClientProtocol.
+ GET_STATS_PENDING_DELETION_BLOCKS_IDX);
}
/**
@@ -1996,7 +2007,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @throws IOException
*/
public long getUnderReplicatedBlocksCount() throws IOException {
- return callGetStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX];
+ return getStateByIndex(ClientProtocol.
+ GET_STATS_UNDER_REPLICATED_IDX);
}
/**
@@ -2004,7 +2016,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @throws IOException
*/
public long getCorruptBlocksCount() throws IOException {
- return callGetStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX];
+ return getStateByIndex(ClientProtocol.
+ GET_STATS_CORRUPT_BLOCKS_IDX);
}
/**
@@ -2014,7 +2027,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
* @throws IOException
*/
public long getBytesInFutureBlocks() throws IOException {
- return callGetStats()[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX];
+ return getStateByIndex(ClientProtocol.
+ GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 551e2e2..d042a53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1115,6 +1115,15 @@ public class DistributedFileSystem extends FileSystem {
}
/**
+ * Returns count of blocks pending on deletion.
+ *
+ * @throws IOException
+ */
+ public long getPendingDeletionBlocksCount() throws IOException {
+ return dfs.getPendingDeletionBlocksCount();
+ }
+
+ /**
* Returns count of blocks with replication factor 1 and have
* lost the only replica.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index e6a99fe..3e82eb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -714,7 +714,8 @@ public interface ClientProtocol {
int GET_STATS_MISSING_BLOCKS_IDX = 5;
int GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX = 6;
int GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX = 7;
- int STATS_ARRAY_LENGTH = 8;
+ int GET_STATS_PENDING_DELETION_BLOCKS_IDX = 8;
+ int STATS_ARRAY_LENGTH = 9;
/**
* Get a set of statistics about the filesystem.
@@ -729,6 +730,7 @@ public interface ClientProtocol {
* <li> [6] contains number of blocks which have replication factor
* 1 and have lost the only replica. </li>
* <li> [7] contains number of bytes that are at risk for deletion. </li>
+ * <li> [8] contains number of pending deletion blocks. </li>
* </ul>
* Use public constants like {@link #GET_STATS_CAPACITY_IDX} in place of
* actual numbers to index into the array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 1e48a69..6989cb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -1562,6 +1562,8 @@ public class PBHelperClient {
res.getMissingReplOneBlocks();
result[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX] =
res.hasBlocksInFuture() ? res.getBlocksInFuture() : 0;
+ result[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX] =
+ res.getPendingDeletionBlocks();
return result;
}
@@ -1931,6 +1933,11 @@ public class PBHelperClient {
result.setBlocksInFuture(
fsStats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX]);
}
+ if (fsStats.length >=
+ ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX + 1) {
+ result.setPendingDeletionBlocks(
+ fsStats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX]);
+ }
return result.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 6a1f777..88f63d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -307,6 +307,7 @@ message GetFsStatsResponseProto {
required uint64 missing_blocks = 6;
optional uint64 missing_repl_one_blocks = 7;
optional uint64 blocks_in_future = 8;
+ optional uint64 pending_deletion_blocks = 9;
}
enum DatanodeReportTypeProto { // type of the datanode report
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 74b8aac..49b4d8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -60,6 +60,9 @@ Trunk (Unreleased)
IMPROVEMENTS
+ HDFS-9653. Added blocks pending deletion report to dfsadmin.
+ (Weiwei Yang via eyang)
+
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
(Junping Du via llu)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 7546b1a..b8d3043 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -184,6 +184,7 @@ class HeartbeatManager implements DatanodeStatistics {
-1L,
-1L,
-1L,
+ -1L,
-1L};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index abc0614..afa41c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3809,6 +3809,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
getMissingReplOneBlocksCount();
stats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX] =
blockManager.getBytesInFuture();
+ stats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX] =
+ blockManager.getPendingDeletionBlocksCount();
return stats;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 3b40608..5da3bc5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -502,6 +502,8 @@ public class DFSAdmin extends FsShell {
dfs.getMissingBlocksCount());
System.out.println("Missing blocks (with replication factor 1): " +
dfs.getMissingReplOneBlocksCount());
+ System.out.println("Pending deletion blocks: " +
+ dfs.getPendingDeletionBlocksCount());
System.out.println();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a2bc0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
index a588a73..7500544 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
@@ -17,16 +17,19 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import java.lang.reflect.Method;
import java.text.SimpleDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
@@ -88,15 +91,31 @@ public class TestPendingInvalidateBlock {
Assert.assertEquals(0, cluster.getNamesystem().getBlocksTotal());
Assert.assertEquals(REPLICATION, cluster.getNamesystem()
.getPendingDeletionBlocks());
+ Assert.assertEquals(REPLICATION,
+ dfs.getPendingDeletionBlocksCount());
Thread.sleep(6000);
Assert.assertEquals(0, cluster.getNamesystem().getBlocksTotal());
Assert.assertEquals(0, cluster.getNamesystem().getPendingDeletionBlocks());
+ Assert.assertEquals(0, dfs.getPendingDeletionBlocksCount());
long nnStarted = cluster.getNamesystem().getNNStartedTimeInMillis();
long blockDeletionStartTime = cluster.getNamesystem()
.getBlockDeletionStartTime();
Assert.assertTrue(String.format(
"Expect blockDeletionStartTime = %d > nnStarted = %d.",
blockDeletionStartTime, nnStarted), blockDeletionStartTime > nnStarted);
+
+ // test client protocol compatibility
+ Method method = DFSClient.class.
+ getDeclaredMethod("getStateByIndex", int.class);
+ method.setAccessible(true);
+ // get number of pending deletion blocks by its index
+ long validState = (Long) method.invoke(dfs.getClient(),
+ ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX);
+ // get an out of index value
+ long invalidState = (Long) method.invoke(dfs.getClient(),
+ ClientProtocol.STATS_ARRAY_LENGTH);
+ Assert.assertEquals(0, validState);
+ Assert.assertEquals(-1, invalidState);
}
/**
[40/50] [abbrv] hadoop git commit: YARN-4371. "yarn application
-kill" should take multiple application ids. Contributed by Sunil G
Posted by ka...@apache.org.
YARN-4371. "yarn application -kill" should take multiple application ids. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e91e8b71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e91e8b71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e91e8b71
Branch: refs/heads/YARN-1011
Commit: e91e8b711c68273460b36557fc37fdfc86be097b
Parents: d22c423
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jan 22 21:36:15 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jan 22 21:36:15 2016 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 57 ++++++--
.../hadoop/yarn/client/cli/TestYarnCLI.java | 141 +++++++++++++++++--
3 files changed, 183 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e91e8b71/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a4f9874..0ceb905 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -733,6 +733,9 @@ Release 2.8.0 - UNRELEASED
YARN-4524. Cleanup AppSchedulingInfo. (Karthik Kambatla via wangda)
+ YARN-4371. "yarn application -kill" should take multiple application ids
+ (Sunil G via jlowe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e91e8b71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 77e0688..caa4d46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -23,6 +23,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@@ -103,7 +104,6 @@ public class ApplicationCLI extends YarnCLI {
+ "Supports optional use of -appTypes to filter applications "
+ "based on application type, "
+ "and -appStates to filter applications based on application state.");
- opts.addOption(KILL_CMD, true, "Kills the application.");
opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a "
+ "different queue.");
opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to"
@@ -127,7 +127,12 @@ public class ApplicationCLI extends YarnCLI {
opts.addOption(UPDATE_PRIORITY, true,
"update priority of an application. ApplicationId can be"
+ " passed using 'appId' option.");
- opts.getOption(KILL_CMD).setArgName("Application ID");
+ Option killOpt = new Option(KILL_CMD, true, "Kills the application. "
+ + "Set of applications can be provided separated with space");
+ killOpt.setValueSeparator(' ');
+ killOpt.setArgs(Option.UNLIMITED_VALUES);
+ killOpt.setArgName("Application ID");
+ opts.addOption(killOpt);
opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID");
opts.getOption(QUEUE_CMD).setArgName("Queue Name");
opts.getOption(STATUS_CMD).setArgName("Application ID");
@@ -239,15 +244,11 @@ public class ApplicationCLI extends YarnCLI {
listContainers(cliParser.getOptionValue(LIST_CMD));
}
} else if (cliParser.hasOption(KILL_CMD)) {
- if (args.length != 3) {
+ if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
printUsage(title, opts);
return exitCode;
}
- try{
- killApplication(cliParser.getOptionValue(KILL_CMD));
- } catch (ApplicationNotFoundException e) {
- return exitCode;
- }
+ return killApplication(cliParser.getOptionValues(KILL_CMD));
} else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) {
if (!cliParser.hasOption(QUEUE_CMD)) {
printUsage(title, opts);
@@ -482,6 +483,30 @@ public class ApplicationCLI extends YarnCLI {
}
/**
+ * Kills applications with the application id as appId
+ *
+ * @param Array of applicationIds
+ * @return errorCode
+ * @throws YarnException
+ * @throws IOException
+ */
+ private int killApplication(String[] applicationIds) throws YarnException,
+ IOException {
+ int returnCode = -1;
+ for (String applicationId : applicationIds) {
+ try {
+ killApplication(applicationId);
+ returnCode = 0;
+ } catch (ApplicationNotFoundException e) {
+ // Suppress all ApplicationNotFoundException for now.
+ continue;
+ }
+ }
+
+ return returnCode;
+ }
+
+ /**
* Kills the application with the application id as appId
*
* @param applicationId
@@ -726,4 +751,20 @@ public class ApplicationCLI extends YarnCLI {
+ " as application is in final states");
}
}
+
+ @SuppressWarnings("unchecked")
+ private boolean hasAnyOtherCLIOptions(CommandLine cliParser, Options opts,
+ String excludeOption) {
+ Collection<Option> ops = opts.getOptions();
+ for (Option op : ops) {
+ // Skip exclude option from the option list
+ if (op.getOpt().equals(excludeOption)) {
+ continue;
+ }
+ if (cliParser.hasOption(op.getOpt())) {
+ return true;
+ }
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e91e8b71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 95ccccf..3783aac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -745,14 +745,6 @@ public class TestYarnCLI {
sysOutStream.toString());
sysOutStream.reset();
- ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
- result = cli.run(
- new String[] {"application", "-kill", applicationId.toString(), "args" });
- verify(spyCli).printUsage(any(String.class), any(Options.class));
- Assert.assertEquals(createApplicationCLIHelpMessage(),
- sysOutStream.toString());
-
- sysOutStream.reset();
NodeId nodeId = NodeId.newInstance("host0", 0);
result = cli.run(
new String[] { "application", "-status", nodeId.toString(), "args" });
@@ -878,7 +870,134 @@ public class TestYarnCLI {
Assert.fail("Unexpected exception: " + e);
}
}
-
+
+ @Test
+ public void testKillApplications() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId1 = ApplicationId.newInstance(1234, 5);
+ ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
+ ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
+ ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
+
+ // Test Scenario 1: Both applications are FINISHED.
+ ApplicationReport newApplicationReport1 = ApplicationReport.newInstance(
+ applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
+ applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.34344f, "YARN", null);
+ when(client.getApplicationReport(applicationId1)).thenReturn(
+ newApplicationReport1);
+ when(client.getApplicationReport(applicationId2)).thenReturn(
+ newApplicationReport2);
+ int result = cli.run(new String[]{"application", "-kill",
+ applicationId1.toString() + " " + applicationId2.toString()});
+ assertEquals(0, result);
+ verify(client, times(0)).killApplication(applicationId1);
+ verify(client, times(0)).killApplication(applicationId2);
+ verify(sysOut).println(
+ "Application " + applicationId1 + " has already finished ");
+ verify(sysOut).println(
+ "Application " + applicationId2 + " has already finished ");
+
+ // Test Scenario 2: Both applications are RUNNING.
+ ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
+ applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ ApplicationReport newApplicationReport4 = ApplicationReport.newInstance(
+ applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null);
+ when(client.getApplicationReport(applicationId1)).thenReturn(
+ newApplicationReport3);
+ when(client.getApplicationReport(applicationId2)).thenReturn(
+ newApplicationReport4);
+ result = cli.run(new String[]{"application", "-kill",
+ applicationId1.toString() + " " + applicationId2.toString()});
+ assertEquals(0, result);
+ verify(client).killApplication(applicationId1);
+ verify(client).killApplication(applicationId2);
+ verify(sysOut).println(
+ "Killing application application_1234_0005");
+ verify(sysOut).println(
+ "Killing application application_1234_0006");
+
+ // Test Scenario 3: Both applications are not present.
+ doThrow(new ApplicationNotFoundException("Application with id '"
+ + applicationId3 + "' doesn't exist in RM.")).when(client)
+ .getApplicationReport(applicationId3);
+ doThrow(new ApplicationNotFoundException("Application with id '"
+ + applicationId4 + "' doesn't exist in RM.")).when(client)
+ .getApplicationReport(applicationId4);
+ result = cli.run(new String[]{"application", "-kill",
+ applicationId3.toString() + " " + applicationId4.toString()});
+ Assert.assertNotEquals(0, result);
+ verify(sysOut).println(
+ "Application with id 'application_1234_0007' doesn't exist in RM.");
+ verify(sysOut).println(
+ "Application with id 'application_1234_0008' doesn't exist in RM.");
+
+ // Test Scenario 4: one application is not present and other RUNNING
+ doThrow(new ApplicationNotFoundException("Application with id '"
+ + applicationId3 + "' doesn't exist in RM.")).when(client)
+ .getApplicationReport(applicationId3);
+ ApplicationReport newApplicationReport5 = ApplicationReport.newInstance(
+ applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null);
+ when(client.getApplicationReport(applicationId1)).thenReturn(
+ newApplicationReport5);
+ result = cli.run(new String[]{"application", "-kill",
+ applicationId3.toString() + " " + applicationId1.toString()});
+ Assert.assertEquals(0, result);
+
+ // Test Scenario 5: kill operation with some other command.
+ sysOutStream.reset();
+ result = cli.run(new String[]{"application", "--appStates", "RUNNING",
+ "-kill", applicationId3.toString() + " " + applicationId1.toString()});
+ Assert.assertEquals(-1, result);
+ Assert.assertEquals(createApplicationCLIHelpMessage(),
+ sysOutStream.toString());
+ }
+
+ @Test
+ public void testKillApplicationsOfDifferentEndStates() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId1 = ApplicationId.newInstance(1234, 5);
+ ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
+
+ // Scenario: One application is FINISHED and other is RUNNING.
+ ApplicationReport newApplicationReport5 = ApplicationReport.newInstance(
+ applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ ApplicationReport newApplicationReport6 = ApplicationReport.newInstance(
+ applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null);
+ when(client.getApplicationReport(applicationId1)).thenReturn(
+ newApplicationReport5);
+ when(client.getApplicationReport(applicationId2)).thenReturn(
+ newApplicationReport6);
+ int result = cli.run(new String[]{"application", "-kill",
+ applicationId1.toString() + " " + applicationId2.toString()});
+ assertEquals(0, result);
+ verify(client, times(1)).killApplication(applicationId2);
+ verify(sysOut).println(
+ "Application " + applicationId1 + " has already finished ");
+ verify(sysOut).println("Killing application application_1234_0006");
+ }
+
@Test
public void testMoveApplicationAcrossQueues() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
@@ -1694,7 +1813,9 @@ public class TestYarnCLI {
pw.println(" based on input comma-separated list of");
pw.println(" application types.");
pw.println(" -help Displays help for all commands.");
- pw.println(" -kill <Application ID> Kills the application.");
+ pw.println(" -kill <Application ID> Kills the application. Set of");
+ pw.println(" applications can be provided separated");
+ pw.println(" with space");
pw.println(" -list List applications. Supports optional use");
pw.println(" of -appTypes to filter applications based");
pw.println(" on application type, and -appStates to");
[34/50] [abbrv] hadoop git commit: YARN-4497. RM might fail to
restart when recovering apps whose attempts are missing. (Jun Gong via
rohithsharmaks)
Posted by ka...@apache.org.
YARN-4497. RM might fail to restart when recovering apps whose attempts are missing. (Jun Gong via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6258b33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6258b33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6258b33
Branch: refs/heads/YARN-1011
Commit: d6258b33a7428a0725ead96bc43f4dd444c7c8f1
Parents: 0bae506
Author: rohithsharmaks <ro...@apache.org>
Authored: Fri Jan 22 20:27:38 2016 +0530
Committer: rohithsharmaks <ro...@apache.org>
Committed: Fri Jan 22 20:27:38 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 26 ++++++++-
.../rmapp/attempt/RMAppAttemptImpl.java | 8 +++
.../server/resourcemanager/TestRMRestart.java | 61 ++++++++++++++++++++
4 files changed, 96 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6258b33/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2230b42..b667b5b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -143,6 +143,9 @@ Release 2.9.0 - UNRELEASED
YARN-4578. Directories that are mounted in docker containers need to be more
restrictive/container-specific. (Sidharta Seethana via vvasudev)
+ YARN-4497. RM might fail to restart when recovering apps whose attempts are missing.
+ (Jun Gong via rohithsharmaks)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6258b33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 1a390df..10c9edc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -849,17 +850,32 @@ public class RMAppImpl implements RMApp, Recoverable {
// send the ATS create Event
sendATSCreateEvent(this, this.startTime);
- for(int i=0; i<appState.getAttemptCount(); ++i) {
+ RMAppAttemptImpl preAttempt = null;
+ for (ApplicationAttemptId attemptId :
+ new TreeSet<>(appState.attempts.keySet())) {
// create attempt
- createNewAttempt();
+ createNewAttempt(attemptId);
((RMAppAttemptImpl)this.currentAttempt).recover(state);
+ // If previous attempt is not in final state, it means we failed to store
+ // its final state. We set it to FAILED now because we could not make sure
+ // about its final state.
+ if (preAttempt != null && preAttempt.getRecoveredFinalState() == null) {
+ preAttempt.setRecoveredFinalState(RMAppAttemptState.FAILED);
+ }
+ preAttempt = (RMAppAttemptImpl)currentAttempt;
+ }
+ if (currentAttempt != null) {
+ nextAttemptId = currentAttempt.getAppAttemptId().getAttemptId() + 1;
}
}
private void createNewAttempt() {
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(applicationId, nextAttemptId++);
+ createNewAttempt(appAttemptId);
+ }
+ private void createNewAttempt(ApplicationAttemptId appAttemptId) {
BlacklistManager currentAMBlacklist;
if (currentAttempt != null) {
currentAMBlacklist = currentAttempt.getAMBlacklist();
@@ -1803,4 +1819,10 @@ public class RMAppImpl implements RMApp, Recoverable {
public float getAmBlacklistingDisableThreshold() {
return blacklistDisableThreshold;
}
+
+ @Private
+ @VisibleForTesting
+ public int getNextAttemptId() {
+ return nextAttemptId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6258b33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 8aefe9f..3f45cb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -2113,4 +2113,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
public void updateAMLaunchDiagnostics(String amLaunchDiagnostics) {
this.amLaunchDiagnostics = amLaunchDiagnostics;
}
+
+ public RMAppAttemptState getRecoveredFinalState() {
+ return recoveredFinalState;
+ }
+
+ public void setRecoveredFinalState(RMAppAttemptState finalState) {
+ this.recoveredFinalState = finalState;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6258b33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 625c3b8..3bab88a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -101,6 +101,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreRMDTMa
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -2379,4 +2380,64 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
}
+ @Test(timeout = 60000)
+ public void testRMRestartOnMissingAttempts() throws Exception {
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 5);
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+
+ // start RM
+ MockRM rm1 = createMockRM(conf, memStore);
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // create an app and finish the app.
+ RMApp app0 = rm1.submitApp(200);
+ ApplicationStateData app0State = memStore.getState().getApplicationState()
+ .get(app0.getApplicationId());
+
+ MockAM am0 = launchAndFailAM(app0, rm1, nm1);
+ MockAM am1 = launchAndFailAM(app0, rm1, nm1);
+ MockAM am2 = launchAndFailAM(app0, rm1, nm1);
+ MockAM am3 = launchAM(app0, rm1, nm1);
+
+ // am1 is missed from MemoryRMStateStore
+ memStore.removeApplicationAttemptInternal(am1.getApplicationAttemptId());
+ ApplicationAttemptStateData am2State = app0State.getAttempt(
+ am2.getApplicationAttemptId());
+ // am2's state is not consistent: MemoryRMStateStore just saved its initial
+ // state and failed to store its final state
+ am2State.setState(null);
+
+ // restart rm
+ MockRM rm2 = createMockRM(conf, memStore);
+ rm2.start();
+
+ Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
+ RMApp recoveredApp0 = rm2.getRMContext().getRMApps().values()
+ .iterator().next();
+ Map<ApplicationAttemptId, RMAppAttempt> recoveredAppAttempts
+ = recoveredApp0.getAppAttempts();
+ Assert.assertEquals(3, recoveredAppAttempts.size());
+ Assert.assertEquals(RMAppAttemptState.FAILED,
+ recoveredAppAttempts.get(
+ am0.getApplicationAttemptId()).getAppAttemptState());
+ Assert.assertEquals(RMAppAttemptState.FAILED,
+ recoveredAppAttempts.get(
+ am2.getApplicationAttemptId()).getAppAttemptState());
+ Assert.assertEquals(RMAppAttemptState.LAUNCHED,
+ recoveredAppAttempts.get(
+ am3.getApplicationAttemptId()).getAppAttemptState());
+ Assert.assertEquals(5, ((RMAppImpl)app0).getNextAttemptId());
+ }
+
+ private MockAM launchAndFailAM(RMApp app, MockRM rm, MockNM nm)
+ throws Exception {
+ MockAM am = launchAM(app, rm, nm);
+ nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ am.waitForState(RMAppAttemptState.FAILED);
+ return am;
+ }
}
[46/50] [abbrv] hadoop git commit: Added MAPREDUCE-6614 to 2.8.0 in
CHANGES.txt.
Posted by ka...@apache.org.
Added MAPREDUCE-6614 to 2.8.0 in CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b833297
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b833297
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b833297
Branch: refs/heads/YARN-1011
Commit: 2b833297ceb523d39e683fcd34ed8ab9b5651bcf
Parents: 10a2bc0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jan 25 11:41:19 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jan 25 11:41:19 2016 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b833297/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 5eb6984..ba392c3 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
mapreduce.reduce.skip.proc.count.autoincr in mapred-default.xml.
(Kai Sasaki via aajisaka)
+ MAPREDUCE-6614. Remove unnecessary code in TestMapreduceConfigFields.
+ (Kai Sasaki via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
[29/50] [abbrv] hadoop git commit: YARN-4578. Directories that are
mounted in docker containers need to be more restrictive/container-specific.
Contributed by Sidharta Seethana.
Posted by ka...@apache.org.
YARN-4578. Directories that are mounted in docker containers need to be more restrictive/container-specific. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b41a7e89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b41a7e89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b41a7e89
Branch: refs/heads/YARN-1011
Commit: b41a7e89d1eb8650975ac7092532ed9563ac60f2
Parents: ae9c61f
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Jan 22 14:43:14 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Jan 22 14:43:14 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/LinuxContainerExecutor.java | 4 ++++
.../launcher/ContainerLaunch.java | 11 +++++++++
.../runtime/DockerLinuxContainerRuntime.java | 10 ++++++--
.../runtime/LinuxContainerRuntimeConstants.java | 4 ++++
.../executor/ContainerStartContext.java | 24 ++++++++++++++++++++
.../runtime/TestDockerContainerRuntime.java | 12 ++++++++--
7 files changed, 64 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a7a63b1..77b4eb4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -140,6 +140,9 @@ Release 2.9.0 - UNRELEASED
YARN-4584. RM startup failure when AM attempts greater than max-attempts.
(Bibin A Chundatt via rohithsharmaks)
+ YARN-4578. Directories that are mounted in docker containers need to be more
+ restrictive/container-specific. (Sidharta Seethana via vvasudev)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 8549230..ff82e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -291,6 +291,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
Path containerWorkDir = ctx.getContainerWorkDir();
List<String> localDirs = ctx.getLocalDirs();
List<String> logDirs = ctx.getLogDirs();
+ List<String> containerLocalDirs = ctx.getContainerLocalDirs();
+ List<String> containerLogDirs = ctx.getContainerLogDirs();
Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
verifyUsernamePattern(user);
@@ -375,6 +377,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
+ .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
if (tcCommandFile != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6371b21..64689dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -253,6 +253,7 @@ public class ContainerLaunch implements Callable<Integer> {
+ dirsHandler.getDisksHealthReport(false));
}
+ List<String> containerLocalDirs = new ArrayList<>(localDirs.size());
try {
// /////////// Write out the container-script in the nmPrivate space.
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
@@ -261,6 +262,14 @@ public class ContainerLaunch implements Callable<Integer> {
Path userdir = new Path(usersdir, user);
Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
appDirs.add(new Path(appsdir, appIdStr));
+
+ String containerLocalDir = localDir + Path.SEPARATOR +
+ ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
+ + Path.SEPARATOR
+ + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
+ + Path.SEPARATOR;
+
+ containerLocalDirs.add(containerLocalDir);
}
containerScriptOutStream =
lfs.create(nmPrivateContainerScriptPath,
@@ -317,6 +326,8 @@ public class ContainerLaunch implements Callable<Integer> {
.setContainerWorkDir(containerWorkDir)
.setLocalDirs(localDirs)
.setLogDirs(logDirs)
+ .setContainerLocalDirs(containerLocalDirs)
+ .setContainerLogDirs(containerLogDirs)
.build());
}
} catch (Throwable e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index f8aae81..2dee663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -223,6 +223,12 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
@SuppressWarnings("unchecked")
List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+ @SuppressWarnings("unchecked")
+ List<String> containerLocalDirs = ctx.getExecutionAttribute(
+ CONTAINER_LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ List<String> containerLogDirs = ctx.getExecutionAttribute(
+ CONTAINER_LOG_DIRS);
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -235,10 +241,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.setNetworkType("host")
.setCapabilities(capabilities)
.addMountLocation("/etc/passwd", "/etc/password:ro");
- List<String> allDirs = new ArrayList<>(localDirs);
+ List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.add(containerWorkDir.toString());
- allDirs.addAll(logDirs);
+ allDirs.addAll(containerLogDirs);
for (String dir: allDirs) {
runCommand.addMountLocation(dir, dir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index d2069a9..96ff40d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants {
List.class, "local_dirs");
public static final Attribute<List> LOG_DIRS = Attribute.attribute(
List.class, "log_dirs");
+ public static final Attribute<List> CONTAINER_LOCAL_DIRS = Attribute
+ .attribute(List.class, "container_local_dirs");
+ public static final Attribute<List> CONTAINER_LOG_DIRS = Attribute.attribute(
+ List.class, "container_log_dirs");
public static final Attribute<String> RESOURCES_OPTIONS = Attribute.attribute(
String.class, "resources_options");
public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
index ffcc519..b14a520 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
@@ -45,6 +45,8 @@ public final class ContainerStartContext {
private final Path containerWorkDir;
private final List<String> localDirs;
private final List<String> logDirs;
+ private final List<String> containerLocalDirs;
+ private final List<String> containerLogDirs;
public static final class Builder {
private Container container;
@@ -56,6 +58,8 @@ public final class ContainerStartContext {
private Path containerWorkDir;
private List<String> localDirs;
private List<String> logDirs;
+ private List<String> containerLocalDirs;
+ private List<String> containerLogDirs;
public Builder() {
}
@@ -107,6 +111,16 @@ public final class ContainerStartContext {
return this;
}
+ public Builder setContainerLocalDirs(List<String> containerLocalDirs) {
+ this.containerLocalDirs = containerLocalDirs;
+ return this;
+ }
+
+ public Builder setContainerLogDirs(List<String> containerLogDirs) {
+ this.containerLogDirs = containerLogDirs;
+ return this;
+ }
+
public ContainerStartContext build() {
return new ContainerStartContext(this);
}
@@ -122,6 +136,8 @@ public final class ContainerStartContext {
this.containerWorkDir = builder.containerWorkDir;
this.localDirs = builder.localDirs;
this.logDirs = builder.logDirs;
+ this.containerLocalDirs = builder.containerLocalDirs;
+ this.containerLogDirs = builder.containerLogDirs;
}
public Container getContainer() {
@@ -163,4 +179,12 @@ public final class ContainerStartContext {
public List<String> getLogDirs() {
return Collections.unmodifiableList(this.logDirs);
}
+
+ public List<String> getContainerLocalDirs() {
+ return this.containerLocalDirs;
+ }
+
+ public List<String> getContainerLogDirs() {
+ return this.containerLogDirs;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 7fc0158..6898634 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -81,6 +81,8 @@ public class TestDockerContainerRuntime {
Path pidFilePath;
List<String> localDirs;
List<String> logDirs;
+ List<String> containerLocalDirs;
+ List<String> containerLogDirs;
String resourcesOptions;
ContainerRuntimeContext.Builder builder;
String submittingUser = "anakin";
@@ -123,9 +125,13 @@ public class TestDockerContainerRuntime {
localDirs = new ArrayList<>();
logDirs = new ArrayList<>();
resourcesOptions = "cgroups=none";
+ containerLocalDirs = new ArrayList<>();
+ containerLogDirs = new ArrayList<>();
localDirs.add("/test_local_dir");
logDirs.add("/test_log_dir");
+ containerLocalDirs.add("/test_container_local_dir");
+ containerLogDirs.add("/test_container_log_dir");
builder = new ContainerRuntimeContext
.Builder(container);
@@ -141,6 +147,8 @@ public class TestDockerContainerRuntime {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
+ .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
}
@@ -245,8 +253,8 @@ public class TestDockerContainerRuntime {
.append("bash %8$s/launch_container.sh");
String expectedCommand = String.format(expectedCommandTemplate.toString(),
- containerId, runAsUser, containerWorkDir, localDirs.get(0),
- containerWorkDir, logDirs.get(0), image, containerWorkDir);
+ containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), image, containerWorkDir);
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
[25/50] [abbrv] hadoop git commit: HADOOP-12659. Incorrect usage of
config parameters in token manager of KMS. Contributed by Mingliang Liu.
Posted by ka...@apache.org.
HADOOP-12659. Incorrect usage of config parameters in token manager of KMS. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3427d37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3427d37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3427d37
Branch: refs/heads/YARN-1011
Commit: f3427d3766d7101d0d1c37d6281918551d221ebe
Parents: b4a05c1
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Jan 21 13:51:58 2016 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Jan 21 13:51:58 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../token/delegation/ZKDelegationTokenSecretManager.java | 2 +-
.../security/token/delegation/web/DelegationTokenManager.java | 4 ++--
3 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3427d37/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e5172d5..2fb1173 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1639,6 +1639,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-7161. Remove unnecessary oro package from dependency management
section. (Sean Busbey via aajisaka)
+ HADOOP-12659. Incorrect usage of config parameters in token manager of
+ KMS. (Mingliang Liu via xyao)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3427d37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index da0e6ad..88b81b0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -141,7 +141,7 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
conf.getLong(DelegationTokenManager.MAX_LIFETIME,
DelegationTokenManager.MAX_LIFETIME_DEFAULT) * 1000,
conf.getLong(DelegationTokenManager.RENEW_INTERVAL,
- DelegationTokenManager.RENEW_INTERVAL_DEFAULT * 1000),
+ DelegationTokenManager.RENEW_INTERVAL_DEFAULT) * 1000,
conf.getLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL,
DelegationTokenManager.REMOVAL_SCAN_INTERVAL_DEFAULT) * 1000);
shutdownTimeout = conf.getLong(ZK_DTSM_ZK_SHUTDOWN_TIMEOUT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3427d37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
index 5d86249..221b02a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
@@ -68,8 +68,8 @@ public class DelegationTokenManager {
super(conf.getLong(UPDATE_INTERVAL, UPDATE_INTERVAL_DEFAULT) * 1000,
conf.getLong(MAX_LIFETIME, MAX_LIFETIME_DEFAULT) * 1000,
conf.getLong(RENEW_INTERVAL, RENEW_INTERVAL_DEFAULT) * 1000,
- conf.getLong(REMOVAL_SCAN_INTERVAL,
- REMOVAL_SCAN_INTERVAL_DEFAULT * 1000));
+ conf.getLong(REMOVAL_SCAN_INTERVAL, REMOVAL_SCAN_INTERVAL_DEFAULT)
+ * 1000);
this.tokenKind = tokenKind;
}
[12/50] [abbrv] hadoop git commit: HADOOP-12423. Handle failure of
registering shutdownhook by ShutdownHookManager in static block (Contributed
by Abhishek Agarwal)
Posted by ka...@apache.org.
HADOOP-12423. Handle failure of registering shutdownhook by ShutdownHookManager in static block (Contributed by Abhishek Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/446987e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/446987e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/446987e2
Branch: refs/heads/YARN-1011
Commit: 446987e20aacd870e1bc3f950b2bd4bbda1f9571
Parents: b7372b7
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Jan 21 20:08:24 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Jan 21 20:08:24 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/util/ShutdownHookManager.java | 31 ++++++++++++--------
2 files changed, 21 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/446987e2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 85f304a..e775f1b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1630,6 +1630,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12356. Fix computing CPU usage statistics on Windows.
(Inigo Goiri via wangda)
+ HADOOP-12423. Handle failure of registering shutdownhook by
+ ShutdownHookManager in static block (Abhishek Agarwal via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/446987e2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index 85533db..843454b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -44,22 +44,27 @@ public class ShutdownHookManager {
private static final Log LOG = LogFactory.getLog(ShutdownHookManager.class);
static {
- Runtime.getRuntime().addShutdownHook(
- new Thread() {
- @Override
- public void run() {
- MGR.shutdownInProgress.set(true);
- for (Runnable hook: MGR.getShutdownHooksInOrder()) {
- try {
- hook.run();
- } catch (Throwable ex) {
- LOG.warn("ShutdownHook '" + hook.getClass().getSimpleName() +
- "' failed, " + ex.toString(), ex);
+ try {
+ Runtime.getRuntime().addShutdownHook(
+ new Thread() {
+ @Override
+ public void run() {
+ MGR.shutdownInProgress.set(true);
+ for (Runnable hook: MGR.getShutdownHooksInOrder()) {
+ try {
+ hook.run();
+ } catch (Throwable ex) {
+ LOG.warn("ShutdownHook '" + hook.getClass().getSimpleName() +
+ "' failed, " + ex.toString(), ex);
+ }
}
}
}
- }
- );
+ );
+ } catch (IllegalStateException ex) {
+ // JVM is being shut down. Ignore
+ LOG.warn("Failed to add the ShutdownHook", ex);
+ }
}
/**
[48/50] [abbrv] hadoop git commit: HADOOP-12736.
TestTimedOutTestsListener#testThreadDumpAndDeadlocks sometimes times out.
Contributed by Xiao Chen.
Posted by ka...@apache.org.
HADOOP-12736. TestTimedOutTestsListener#testThreadDumpAndDeadlocks sometimes times out. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64322792
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64322792
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64322792
Branch: refs/heads/YARN-1011
Commit: 643227927a7d7974655627d7e97aae42600692ae
Parents: 736eb17
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jan 25 13:59:25 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jan 25 13:59:25 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/test/TestTimedOutTestsListener.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64322792/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3db68fb..2fc8ab4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -2493,6 +2493,9 @@ Release 2.6.4 - UNRELEASED
HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+ HADOOP-12736. TestTimedOutTestsListener#testThreadDumpAndDeadlocks
+ sometimes times out. (Xiao Chen via aajisaka)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64322792/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
index 62748b4..1334f1c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
@@ -144,7 +144,7 @@ public class TestTimedOutTestsListener {
}
- @Test(timeout=500)
+ @Test(timeout=30000)
public void testThreadDumpAndDeadlocks() throws Exception {
new Deadlock();
String s = null;
[49/50] [abbrv] hadoop git commit: YARN-4592. Remove unused
GetContainerStatus proto. Contributed by Chang Li.
Posted by ka...@apache.org.
YARN-4592. Remove unused GetContainerStatus proto. Contributed by Chang Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56a0c175
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56a0c175
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56a0c175
Branch: refs/heads/YARN-1011
Commit: 56a0c175082b2b62698f65c1769a64224b3fb821
Parents: 6432279
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Jan 25 15:20:29 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Jan 25 15:20:29 2016 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../hadoop-yarn-api/src/main/proto/yarn_service_protos.proto | 7 -------
2 files changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a0c175/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6674194..8ece214 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1325,6 +1325,8 @@ Release 2.8.0 - UNRELEASED
YARN-4614. Fix random failure in TestApplicationPriority#testApplicationPriority
AllocationWithChangeInPriority. (Sunil G via rohithsharmaks)
+ YARN-4592. Remove unused GetContainerStatus proto. (Chang Li via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a0c175/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 115df9a..eae840b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -262,13 +262,6 @@ message StopContainerRequestProto {
message StopContainerResponseProto {
}
-message GetContainerStatusRequestProto {
- optional ContainerIdProto container_id = 1;
-}
-
-message GetContainerStatusResponseProto {
- optional ContainerStatusProto status = 1;
-}
//// bulk API records
message StartContainersRequestProto {
[41/50] [abbrv] hadoop git commit: YARN-4598. Invalid event:
RESOURCE_FAILED at CONTAINER_CLEANEDUP_AFTER_KILL. Contributed by
tangshangwen
Posted by ka...@apache.org.
YARN-4598. Invalid event: RESOURCE_FAILED at CONTAINER_CLEANEDUP_AFTER_KILL. Contributed by tangshangwen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46e5ea81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46e5ea81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46e5ea81
Branch: refs/heads/YARN-1011
Commit: 46e5ea81e0e41646ba02391aea31481ead28aaf8
Parents: e91e8b7
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jan 22 21:55:01 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jan 22 21:55:01 2016 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 ++++++
.../nodemanager/containermanager/container/ContainerImpl.java | 1 +
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e5ea81/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0ceb905..f840a9e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1389,6 +1389,9 @@ Release 2.7.3 - UNRELEASED
YARN-4610. Reservations continue looking for one app causes other apps to
starve (jlowe)
+ YARN-4598. Invalid event: RESOURCE_FAILED at
+ CONTAINER_CLEANEDUP_AFTER_KILL (tangshangwen via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -2278,6 +2281,9 @@ Release 2.6.4 - UNRELEASED
YARN-4581. AHS writer thread leak makes RM crash while RM is recovering.
(sandflee via junping_du)
+ YARN-4598. Invalid event: RESOURCE_FAILED at
+ CONTAINER_CLEANEDUP_AFTER_KILL (tangshangwen via jlowe)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e5ea81/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 3c49489..fb1728a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -328,6 +328,7 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.RESOURCE_FAILED,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE))
[33/50] [abbrv] hadoop git commit: HADOOP-12730. Hadoop streaming
-mapper and -reducer options are wrongly documented as required. Contributed
by Kengo Seki.
Posted by ka...@apache.org.
HADOOP-12730. Hadoop streaming -mapper and -reducer options are wrongly documented as required. Contributed by Kengo Seki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bae506c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bae506c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bae506c
Branch: refs/heads/YARN-1011
Commit: 0bae506c221f9736d08a6fba00ce5a862ab43cce
Parents: 6831634
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jan 22 21:08:31 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Jan 22 21:08:31 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bae506c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2fb1173..36cac2f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1642,6 +1642,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12659. Incorrect usage of config parameters in token manager of
KMS. (Mingliang Liu via xyao)
+ HADOOP-12730. Hadoop streaming -mapper and -reducer options are wrongly
+ documented as required. (Kengo Seki via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bae506c/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
index 0598a35..2223352 100644
--- a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
+++ b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
@@ -96,8 +96,8 @@ The Hadoop streaming command options are listed here:
|:---- |:---- |:---- |
| -input directoryname or filename | Required | Input location for mapper |
| -output directoryname | Required | Output location for reducer |
-| -mapper executable or JavaClassName | Required | Mapper executable |
-| -reducer executable or JavaClassName | Required | Reducer executable |
+| -mapper executable or JavaClassName | Optional | Mapper executable. If not specified, IdentityMapper is used as the default |
+| -reducer executable or JavaClassName | Optional | Reducer executable. If not specified, IdentityReducer is used as the default |
| -file filename | Optional | Make the mapper, reducer, or combiner executable available locally on the compute nodes |
| -inputformat JavaClassName | Optional | Class you supply should return key/value pairs of Text class. If not specified, TextInputFormat is used as the default |
| -outputformat JavaClassName | Optional | Class you supply should take key/value pairs of Text class. If not specified, TextOutputformat is used as the default |
[39/50] [abbrv] hadoop git commit: HDFS-9525. hadoop utilities need
to support provided delegation tokens (HeeSoo Kim via aw)
Posted by ka...@apache.org.
HDFS-9525. hadoop utilities need to support provided delegation tokens (HeeSoo Kim via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d22c4239
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d22c4239
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d22c4239
Branch: refs/heads/YARN-1011
Commit: d22c4239a40a1c7ed49c06038138f0e3f387b4a0
Parents: 95363bc
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jan 22 12:15:22 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jan 22 12:15:22 2016 -0800
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 3 ++
.../hadoop/security/UserGroupInformation.java | 22 +++++++++
.../src/main/resources/core-default.xml | 6 +++
.../security/TestUserGroupInformation.java | 48 +++++++++++++++++++-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 14 +++---
.../hdfs/web/resources/DelegationParam.java | 5 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 5 +-
8 files changed, 92 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index c9f758b..648ad59 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -311,6 +311,9 @@ public class CommonConfigurationKeysPublic {
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY =
"hadoop.security.dns.nameserver";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_TOKEN_FILES =
+ "hadoop.token.files";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 28014bf..d33e1aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.security;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
import java.io.File;
@@ -70,6 +71,7 @@ import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
@@ -821,6 +823,26 @@ public class UserGroupInformation {
}
loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser, realUser);
+ String tokenFileLocation = System.getProperty(HADOOP_TOKEN_FILES);
+ if (tokenFileLocation == null) {
+ tokenFileLocation = conf.get(HADOOP_TOKEN_FILES);
+ }
+ if (tokenFileLocation != null) {
+ for (String tokenFileName:
+ StringUtils.getTrimmedStrings(tokenFileLocation)) {
+ if (tokenFileName.length() > 0) {
+ File tokenFile = new File(tokenFileName);
+ if (tokenFile.exists() && tokenFile.isFile()) {
+ Credentials cred = Credentials.readTokenStorageFile(
+ tokenFile, conf);
+ loginUser.addCredentials(cred);
+ } else {
+ LOG.info("tokenFile("+tokenFileName+") does not exist");
+ }
+ }
+ }
+ }
+
String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
if (fileLocation != null) {
// Load the token storage file and put all of the tokens into the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index bf60a25..c25f49e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -466,6 +466,12 @@ for ldap providers in the same way as above does.
<description>Maps kerberos principals to local user names</description>
</property>
+<property>
+ <name>hadoop.token.files</name>
+ <value></value>
+ <description>List of token cache files that have delegation tokens for hadoop service</description>
+</property>
+
<!-- i/o properties -->
<property>
<name>io.file.buffer.size</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 54cfc2d..6abe78c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -35,6 +36,7 @@ import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Method;
@@ -848,7 +850,9 @@ public class TestUserGroupInformation {
*/
@Test
public void testPrivateTokenExclusion() throws Exception {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation ugi =
+ UserGroupInformation.createUserForTesting(
+ "privateUser", new String[] { "PRIVATEUSERS" });
TestTokenIdentifier tokenId = new TestTokenIdentifier();
Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(
tokenId.getBytes(), "password".getBytes(),
@@ -928,4 +932,46 @@ public class TestUserGroupInformation {
}
}
}
+
+ @Test
+ public void testExternalTokenFiles() throws Exception {
+ StringBuilder tokenFullPathnames = new StringBuilder();
+ String tokenFilenames = "token1,token2";
+ String tokenFiles[] = StringUtils.getTrimmedStrings(tokenFilenames);
+ final File testDir = new File("target",
+ TestUserGroupInformation.class.getName() + "-tmpDir").getAbsoluteFile();
+ String testDirPath = testDir.getAbsolutePath();
+
+ // create path for token files
+ for (String tokenFile: tokenFiles) {
+ if (tokenFullPathnames.length() > 0) {
+ tokenFullPathnames.append(",");
+ }
+ tokenFullPathnames.append(testDirPath).append("/").append(tokenFile);
+ }
+
+ // create new token and store it
+ TestTokenIdentifier tokenId = new TestTokenIdentifier();
+ Credentials cred1 = new Credentials();
+ Token<TestTokenIdentifier> token1 = new Token<TestTokenIdentifier>(
+ tokenId.getBytes(), "password".getBytes(),
+ tokenId.getKind(), new Text("token-service1"));
+ cred1.addToken(token1.getService(), token1);
+ cred1.writeTokenStorageFile(new Path(testDirPath, tokenFiles[0]), conf);
+
+ Credentials cred2 = new Credentials();
+ Token<TestTokenIdentifier> token2 = new Token<TestTokenIdentifier>(
+ tokenId.getBytes(), "password".getBytes(),
+ tokenId.getKind(), new Text("token-service2"));
+ cred2.addToken(token2.getService(), token2);
+ cred2.writeTokenStorageFile(new Path(testDirPath, tokenFiles[1]), conf);
+
+ // set property for token external token files
+ System.setProperty("hadoop.token.files", tokenFullPathnames.toString());
+ UserGroupInformation.setLoginUser(null);
+ UserGroupInformation tokenUgi = UserGroupInformation.getLoginUser();
+ Collection<Token<?>> credsugiTokens = tokenUgi.getTokens();
+ assertTrue(credsugiTokens.contains(token1));
+ assertTrue(credsugiTokens.contains(token2));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index f9c2c6e..d806d55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -238,7 +238,7 @@ public class WebHdfsFileSystem extends FileSystem
// the first getAuthParams() for a non-token op will either get the
// internal token from the ugi or lazy fetch one
protected synchronized Token<?> getDelegationToken() throws IOException {
- if (canRefreshDelegationToken && delegationToken == null) {
+ if (delegationToken == null) {
Token<?> token = tokenSelector.selectToken(
new Text(getCanonicalServiceName()), ugi.getTokens());
// ugi tokens are usually indicative of a task which can't
@@ -248,11 +248,13 @@ public class WebHdfsFileSystem extends FileSystem
LOG.debug("Using UGI token: {}", token);
canRefreshDelegationToken = false;
} else {
- token = getDelegationToken(null);
- if (token != null) {
- LOG.debug("Fetched new token: {}", token);
- } else { // security is disabled
- canRefreshDelegationToken = false;
+ if (canRefreshDelegationToken) {
+ token = getDelegationToken(null);
+ if (token != null) {
+ LOG.debug("Fetched new token: {}", token);
+ } else { // security is disabled
+ canRefreshDelegationToken = false;
+ }
}
}
setDelegationToken(token);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
index 5329580..fda14438 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.web.resources;
-import org.apache.hadoop.security.UserGroupInformation;
-
/** Represents delegation token used for authentication. */
public class DelegationParam extends StringParam {
/** Parameter name. */
@@ -33,8 +31,7 @@ public class DelegationParam extends StringParam {
* @param str a string representation of the parameter value.
*/
public DelegationParam(final String str) {
- super(DOMAIN, UserGroupInformation.isSecurityEnabled()
- && str != null && !str.equals(DEFAULT)? str: null);
+ super(DOMAIN, str != null && !str.equals(DEFAULT)? str: null);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 940fa90..74b8aac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -55,6 +55,9 @@ Trunk (Unreleased)
HDFS-9057. allow/disallow snapshots via webhdfs
(Bramma Reddy Battula via vinayakumarb)
+ HDFS-9525. hadoop utilities need to support provided delegation
+ tokens (HeeSoo Kim via aw)
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
index 2913a97..24c13af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
@@ -195,7 +195,7 @@ public class TestWebHdfsUrl {
checkQueryParams(
new String[]{
GetOpParam.Op.GETFILESTATUS.toQueryString(),
- new UserParam(ugi.getShortUserName()).toString()
+ new DelegationParam(tokenString).toString()
},
fileStatusUrl);
}
@@ -280,8 +280,7 @@ public class TestWebHdfsUrl {
checkQueryParams(
new String[]{
GetOpParam.Op.GETFILESTATUS.toQueryString(),
- new UserParam(ugi.getRealUser().getShortUserName()).toString(),
- new DoAsParam(ugi.getShortUserName()).toString()
+ new DelegationParam(tokenString).toString()
},
fileStatusUrl);
}
[30/50] [abbrv] hadoop git commit: MAPREDUCE-6605. Fix typos
mapreduce.map.skip.proc.count.autoincr and
mapreduce.reduce.skip.proc.count.autoincr in mapred-default.xml. Contributed
by Kai Sasaki.
Posted by ka...@apache.org.
MAPREDUCE-6605. Fix typos mapreduce.map.skip.proc.count.autoincr and mapreduce.reduce.skip.proc.count.autoincr in mapred-default.xml. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5c8c857
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5c8c857
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5c8c857
Branch: refs/heads/YARN-1011
Commit: f5c8c857beea5b1c0a652f3097c15c06c421fbaa
Parents: b41a7e8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jan 22 18:35:43 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Jan 22 18:35:43 2016 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 ++++
.../src/main/resources/mapred-default.xml | 24 ++++++++++++++++++++
2 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5c8c857/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 373b4bb..5eb6984 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -691,6 +691,10 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6601. Fix typo in Job#setUseNewAPI. (Kai Sasaki via aajisaka)
+ MAPREDUCE-6605. Fix typos mapreduce.map.skip.proc.count.autoincr and
+ mapreduce.reduce.skip.proc.count.autoincr in mapred-default.xml.
+ (Kai Sasaki via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5c8c857/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 962584c..efbbf53 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -912,6 +912,18 @@
</property>
<property>
+ <name>mapreduce.map.skip.proc-count.auto-incr</name>
+ <value>true</value>
+ <description>The flag which if set to true,
+ SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS is incremented by
+ MapRunner after invoking the map function. This value must be set
+ to false for applications which process the records asynchronously
+ or buffer the input records. For example streaming. In such cases
+ applications should increment this counter on their own.
+ </description>
+ </property>
+
+ <property>
<name>mapreduce.reduce.skip.maxgroups</name>
<value>0</value>
<description> The number of acceptable skip groups surrounding the bad
@@ -927,6 +939,18 @@
</property>
<property>
+ <name>mapreduce.reduce.skip.proc-count.auto-incr</name>
+ <value>true</value>
+ <description>The flag which if set to true.
+ SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS is incremented by framework
+ after invoking the reduce function. This value must be set to false for
+ applications which process the records asynchronously or buffer the input
+ records. For example streaming. In such cases applications should increment
+ this counter on their own.
+ </description>
+ </property>
+
+ <property>
<name>mapreduce.ifile.readahead</name>
<value>true</value>
<description>Configuration key to enable/disable IFile readahead.
[14/50] [abbrv] hadoop git commit: YARN-4492. Add documentation for
preemption supported in Capacity scheduler. Contributed by Naganarasimha G R
Posted by ka...@apache.org.
YARN-4492. Add documentation for preemption supported in Capacity scheduler. Contributed by Naganarasimha G R
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f858ac24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f858ac24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f858ac24
Branch: refs/heads/YARN-1011
Commit: f858ac24269c101bdee1f7b7b2566bdbda2b4ecd
Parents: 3616c7b
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Jan 21 15:37:51 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Jan 21 15:38:40 2016 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../src/site/markdown/CapacityScheduler.md | 71 ++++++++++++++------
2 files changed, 52 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f858ac24/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bd467ea..4e36c90 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1302,6 +1302,9 @@ Release 2.7.3 - UNRELEASED
YARN-4287. Capacity Scheduler: Rack Locality improvement (Nathan Roberts via wangda)
+ YARN-4492. Add documentation for preemption supported in Capacity
+ scheduler (Naganarasimha G R via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f858ac24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index cd98dc0..b43a032 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -22,9 +22,10 @@ Hadoop: Capacity Scheduler
* [Setting up `ResourceManager` to use `CapacityScheduler`](#Setting_up_ResourceManager_to_use_CapacityScheduler`)
* [Setting up queues](#Setting_up_queues)
* [Queue Properties](#Queue_Properties)
+ * [Setup for application priority](#Setup_for_application_priority.)
+ * [Capacity Scheduler container preemption](#Capacity_Scheduler_container_preemption)
* [Other Properties](#Other_Properties)
* [Reviewing the configuration of the CapacityScheduler](#Reviewing_the_configuration_of_the_CapacityScheduler)
- * [Setup for application priority](#Setup_for_application_priority.)
* [Changing Queue Configuration](#Changing_Queue_Configuration)
Purpose
@@ -58,7 +59,7 @@ The `CapacityScheduler` supports the following features:
* **Security** - Each queue has strict ACLs which controls which users can submit applications to individual queues. Also, there are safe-guards to ensure that users cannot view and/or modify applications from other users. Also, per-queue and system administrator roles are supported.
-* **Elasticity** - Free resources can be allocated to any queue beyond it's capacity. When there is demand for these resources from queues running below capacity at a future point in time, as tasks scheduled on these resources complete, they will be assigned to applications on queues running below the capacity (pre-emption is not supported). This ensures that resources are available in a predictable and elastic manner to queues, thus preventing artifical silos of resources in the cluster which helps utilization.
+* **Elasticity** - Free resources can be allocated to any queue beyond its capacity. When there is demand for these resources from queues running below capacity at a future point in time, as tasks scheduled on these resources complete, they will be assigned to applications on queues running below the capacity (pre-emption is also supported). This ensures that resources are available in a predictable and elastic manner to queues, thus preventing artifical silos of resources in the cluster which helps utilization.
* **Multi-tenancy** - Comprehensive set of limits are provided to prevent a single application, user and queue from monopolizing resources of the queue or the cluster as a whole to ensure that the cluster isn't overwhelmed.
@@ -180,51 +181,77 @@ Example:
</property>
```
-###Other Properties
+###Setup for application priority.
- * Resource Calculator
+ Application priority works only along with FIFO ordering policy. Default ordering policy is FIFO.
+
+ Default priority for an application can be at cluster level and queue level.
+
+ * Cluster-level priority : Any application submitted with a priority greater than the cluster-max priority will have its priority reset to the cluster-max priority.
+ $HADOOP_HOME/etc/hadoop/yarn-site.xml is the configuration file for cluster-max priority.
| Property | Description |
|:---- |:---- |
-| `yarn.scheduler.capacity.resource-calculator` | The ResourceCalculator implementation to be used to compare Resources in the scheduler. The default i.e. org.apache.hadoop.yarn.util.resource.DefaultResourseCalculator only uses Memory while DominantResourceCalculator uses Dominant-resource to compare multi-dimensional resources such as Memory, CPU etc. A Java ResourceCalculator class name is expected. |
+| `yarn.cluster.max-application-priority` | Defines maximum application priority in a cluster. |
- * Data Locality
+ * Leaf Queue-level priority : Each leaf queue provides default priority by the administrator. The queue's default priority will be used for any application submitted without a specified priority.
+ $HADOOP_HOME/etc/hadoop/capacity-scheduler.xml is the configuration file for queue-level priority.
| Property | Description |
|:---- |:---- |
-| `yarn.scheduler.capacity.node-locality-delay` | Number of missed scheduling opportunities after which the CapacityScheduler attempts to schedule rack-local containers. Typically, this should be set to number of nodes in the cluster. By default is setting approximately number of nodes in one rack which is 40. Positive integer value is expected. |
+| `yarn.scheduler.capacity.root.<leaf-queue-path>.default-application-priority` | Defines default application priority in a leaf queue. |
-###Reviewing the configuration of the CapacityScheduler
+**Note:** Priority of an application will not be changed when application is moved to different queue.
- Once the installation and configuration is completed, you can review it after starting the YARN cluster from the web-ui.
+### Capacity Scheduler container preemption
- * Start the YARN cluster in the normal manner.
+ The `CapacityScheduler` supports preemption of container from the queues whose resource usage is more than their guaranteed capacity. The following configuration parameters need to be enabled in yarn-site.xml for supporting preemption of application containers.
- * Open the `ResourceManager` web UI.
+| Property | Description |
+|:---- |:---- |
+| `yarn.resourcemanager.scheduler.monitor.enable` | Enable a set of periodic monitors (specified in yarn.resourcemanager.scheduler.monitor.policies) that affect the scheduler. Default value is false. |
+| `yarn.resourcemanager.scheduler.monitor.policies` | The list of SchedulingEditPolicy classes that interact with the scheduler. Configured policies need to be compatible with the scheduler. Default value is `org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy` which is compatible with `CapacityScheduler` |
- * The */scheduler* web-page should show the resource usages of individual queues.
+The following configuration parameters can be configured in yarn-site.xml to control the preemption of containers when `ProportionalCapacityPreemptionPolicy` class is configured for `yarn.resourcemanager.scheduler.monitor.policies`
-###Setup for application priority.
+| Property | Description |
+|:---- |:---- |
+| `yarn.resourcemanager.monitor.capacity.preemption.observe_only` | If true, run the policy but do not affect the cluster with preemption and kill events. Default value is false |
+| `yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval` | Time in milliseconds between invocations of this ProportionalCapacityPreemptionPolicy policy. Default value is 3000 |
+| `yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill` | Time in milliseconds between requesting a preemption from an application and killing the container. Default value is 15000 |
+| `yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round` | Maximum percentage of resources preempted in a single round. By controlling this value one can throttle the pace at which containers are reclaimed from the cluster. After computing the total desired preemption, the policy scales it back within this limit. Default value is `0.1` |
+| `yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity` | Maximum amount of resources above the target capacity ignored for preemption. This defines a deadzone around the target capacity that helps prevent thrashing and oscillations around the computed target balance. High values would slow the time to capacity and (absent natural.completions) it might prevent convergence to guaranteed capacity. Default value is `0.1` |
+| `yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor` | Given a computed preemption target, account for containers naturally expiring and preempt only this percentage of the delta. This determines the rate of geometric convergence into the deadzone (`MAX_IGNORED_OVER_CAPACITY`). For example, a termination factor of 0.5 will reclaim almost 95% of resources within 5 * #`WAIT_TIME_BEFORE_KILL`, even absent natural termination. Default value is `0.2` |
- Application priority works only along with FIFO ordering policy. Default ordering policy is FIFO.
+ The `CapacityScheduler` supports the following configurations in capacity-scheduler.xml to control the preemption of application containers submitted to a queue.
- Default priority for an application can be at cluster level and queue level.
+| Property | Description |
+|:---- |:---- |
+| `yarn.scheduler.capacity.<queue-path>.disable_preemption` | This configuration can be set to `true` to selectively disable preemption of application containers submitted to a given queue. This property applies only when system wide preemption is enabled by configuring `yarn.resourcemanager.scheduler.monitor.enable` to *true* and `yarn.resourcemanager.scheduler.monitor.policies` to *ProportionalCapacityPreemptionPolicy*. If this property is not set for a queue, then the property value is inherited from the queue's parent. Default value is false.
- * Cluster-level priority : Any application that is submitted with a priority is greater then cluster max priority then application priority will be reset with cluster-max priority.
- $HADOOP_HOME/etc/hadoop/yarn-site.xml is the configuration file for cluster-max priority.
+###Other Properties
+
+ * Resource Calculator
| Property | Description |
|:---- |:---- |
-| `yarn.cluster.max-application-priority` | Defines maximum application priority in a cluster. |
+| `yarn.scheduler.capacity.resource-calculator` | The ResourceCalculator implementation to be used to compare Resources in the scheduler. The default i.e. org.apache.hadoop.yarn.util.resource.DefaultResourseCalculator only uses Memory while DominantResourceCalculator uses Dominant-resource to compare multi-dimensional resources such as Memory, CPU etc. A Java ResourceCalculator class name is expected. |
- * Leaf Queue-level priority : Each leaf queue provides default priority by the administrator. If application is not submitted with any priority, then default priority of the queue will be taken in account.
- $HADOOP_HOME/etc/hadoop/capacity-scheduler.xml is the configuration file for queue-level priority.
+ * Data Locality
| Property | Description |
|:---- |:---- |
-| `yarn.scheduler.capacity.root.<leaf-queue-path>.default-application-priority` | Defines default application priority in a leaf queue. |
+| `yarn.scheduler.capacity.node-locality-delay` | Number of missed scheduling opportunities after which the CapacityScheduler attempts to schedule rack-local containers. Typically, this should be set to number of nodes in the cluster. By default is setting approximately number of nodes in one rack which is 40. Positive integer value is expected. |
-**Note:** Priority of an application will not be changed when application is moved to different queue.
+###Reviewing the configuration of the CapacityScheduler
+
+ Once the installation and configuration is completed, you can review it after starting the YARN cluster from the web-ui.
+
+ * Start the YARN cluster in the normal manner.
+
+ * Open the `ResourceManager` web UI.
+
+ * The */scheduler* web-page should show the resource usages of individual queues.
Changing Queue Configuration
----------------------------
[50/50] [abbrv] hadoop git commit: HADOOP-12715.
TestValueQueue#testgetAtMostPolicyALL fails intermittently. Contributed by
Xiao Chen.
Posted by ka...@apache.org.
HADOOP-12715. TestValueQueue#testgetAtMostPolicyALL fails intermittently. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6eacdea0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6eacdea0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6eacdea0
Branch: refs/heads/YARN-1011
Commit: 6eacdea0e475b4fff91cedce5005a7c11749cf64
Parents: 56a0c17
Author: Walter Su <wa...@apache.org>
Authored: Mon Jan 25 19:30:04 2016 +0800
Committer: Walter Su <wa...@apache.org>
Committed: Mon Jan 25 19:30:04 2016 +0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/crypto/key/TestValueQueue.java | 65 +++++++++++++++-----
2 files changed, 52 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eacdea0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2fc8ab4..9606296 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -2496,6 +2496,9 @@ Release 2.6.4 - UNRELEASED
HADOOP-12736. TestTimedOutTestsListener#testThreadDumpAndDeadlocks
sometimes times out. (Xiao Chen via aajisaka)
+ HADOOP-12715. TestValueQueue#testgetAtMostPolicyALL fails intermittently.
+ (Xiao Chen via waltersu4549)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eacdea0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
index 8e3a093..5eae9a0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
@@ -19,18 +19,24 @@ package org.apache.hadoop.crypto.key;
import java.io.IOException;
import java.util.Queue;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.crypto.key.kms.ValueQueue;
import org.apache.hadoop.crypto.key.kms.ValueQueue.QueueRefiller;
import org.apache.hadoop.crypto.key.kms.ValueQueue.SyncGenerationPolicy;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.base.Supplier;
import com.google.common.collect.Sets;
public class TestValueQueue {
+ Logger LOG = LoggerFactory.getLogger(TestValueQueue.class);
private static class FillInfo {
final int num;
@@ -60,7 +66,7 @@ public class TestValueQueue {
/**
* Verifies that Queue is initially filled to "numInitValues"
*/
- @Test
+ @Test(timeout=30000)
public void testInitFill() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
@@ -74,7 +80,7 @@ public class TestValueQueue {
/**
* Verifies that Queue is initialized (Warmed-up) for provided keys
*/
- @Test
+ @Test(timeout=30000)
public void testWarmUp() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
@@ -97,7 +103,7 @@ public class TestValueQueue {
* Verifies that the refill task is executed after "checkInterval" if
* num values below "lowWatermark"
*/
- @Test
+ @Test(timeout=30000)
public void testRefill() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
@@ -116,7 +122,7 @@ public class TestValueQueue {
* Verifies that the No refill Happens after "checkInterval" if
* num values above "lowWatermark"
*/
- @Test
+ @Test(timeout=30000)
public void testNoRefill() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
@@ -131,29 +137,56 @@ public class TestValueQueue {
/**
* Verify getAtMost when SyncGeneration Policy = ALL
*/
- @Test
+ @Test(timeout=30000)
public void testgetAtMostPolicyALL() throws Exception {
MockFiller filler = new MockFiller();
- ValueQueue<String> vq =
+ final ValueQueue<String> vq =
new ValueQueue<String>(10, 0.1f, 300, 1,
SyncGenerationPolicy.ALL, filler);
Assert.assertEquals("test", vq.getNext("k1"));
Assert.assertEquals(1, filler.getTop().num);
- // Drain completely
- Assert.assertEquals(10, vq.getAtMost("k1", 10).size());
- // Synchronous call
- Assert.assertEquals(10, filler.getTop().num);
- // Ask for more... return all
- Assert.assertEquals(19, vq.getAtMost("k1", 19).size());
+
+ // Synchronous call:
+ // 1. Synchronously fill returned list
+ // 2. Start another async task to fill the queue in the cache
+ Assert.assertEquals("Failed in sync call.", 10,
+ vq.getAtMost("k1", 10).size());
+ Assert.assertEquals("Sync call filler got wrong number.", 10,
+ filler.getTop().num);
+
+ // Wait for the async task to finish
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ int size = vq.getSize("k1");
+ if (size != 10) {
+ LOG.info("Current ValueQueue size is " + size);
+ return false;
+ }
+ return true;
+ } catch (ExecutionException e) {
+ LOG.error("Exception when getSize.", e);
+ return false;
+ }
+ }
+ }, 100, 3000);
+ Assert.assertEquals("Failed in async call.", 10, filler.getTop().num);
+
+ // Drain completely after filled by the async thread
+ Assert.assertEquals("Failed to drain completely after async.", 10,
+ vq.getAtMost("k1", 10).size());
// Synchronous call (No Async call since num > lowWatermark)
- Assert.assertEquals(19, filler.getTop().num);
+ Assert.assertEquals("Failed to get all 19.", 19,
+ vq.getAtMost("k1", 19).size());
+ Assert.assertEquals("Failed in sync call.", 19, filler.getTop().num);
vq.shutdown();
}
/**
* Verify getAtMost when SyncGeneration Policy = ALL
*/
- @Test
+ @Test(timeout=30000)
public void testgetAtMostPolicyATLEAST_ONE() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
@@ -171,7 +204,7 @@ public class TestValueQueue {
/**
* Verify getAtMost when SyncGeneration Policy = LOW_WATERMARK
*/
- @Test
+ @Test(timeout=30000)
public void testgetAtMostPolicyLOW_WATERMARK() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
@@ -188,7 +221,7 @@ public class TestValueQueue {
vq.shutdown();
}
- @Test
+ @Test(timeout=30000)
public void testDrain() throws Exception {
MockFiller filler = new MockFiller();
ValueQueue<String> vq =
[17/50] [abbrv] hadoop git commit: HADOOP-7161. Remove unnecessary
oro package from dependency management section. Contributed by Sean Busbey.
Posted by ka...@apache.org.
HADOOP-7161. Remove unnecessary oro package from dependency management section. Contributed by Sean Busbey.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ac39ca7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ac39ca7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ac39ca7
Branch: refs/heads/YARN-1011
Commit: 2ac39ca762fd7e47b7b9a7b3308b31138b0f59be
Parents: 3ac9511
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Jan 22 02:45:22 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Jan 22 02:45:22 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-project/pom.xml | 5 -----
2 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac39ca7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e775f1b..19ef66e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1633,6 +1633,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12423. Handle failure of registering shutdownhook by
ShutdownHookManager in static block (Abhishek Agarwal via vinayakumarb)
+ HADOOP-7161. Remove unnecessary oro package from dependency management
+ section. (Sean Busbey via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac39ca7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 36d735a..3f4e93f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -735,11 +735,6 @@
<version>3.1.1</version>
</dependency>
<dependency>
- <groupId>oro</groupId>
- <artifactId>oro</artifactId>
- <version>2.0.8</version>
- </dependency>
- <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>${jackson.version}</version>
[26/50] [abbrv] hadoop git commit: HDFS-9601.
NNThroughputBenchmark.BlockReportStats should handle
NotReplicatedYetException on adding block (iwasakims)
Posted by ka...@apache.org.
HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle NotReplicatedYetException on adding block (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2ffcc29
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2ffcc29
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2ffcc29
Branch: refs/heads/YARN-1011
Commit: b2ffcc2915838cf2e35544cfbb0286984a435205
Parents: f3427d3
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Fri Jan 22 12:28:38 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Fri Jan 22 12:28:38 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/namenode/NNThroughputBenchmark.java | 34 +++++++++++++++++++-
2 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2ffcc29/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0ded0c4..140be77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -948,6 +948,9 @@ Release 2.9.0 - UNRELEASED
BUG FIXES
+ HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle
+ NotReplicatedYetException on adding block (iwasakims)
+
HDFS-9621. getListing wrongly associates Erasure Coding policy to pre-existing
replicated files under an EC directory. (jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2ffcc29/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 8a594ed..42cb72f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups;
@@ -1181,7 +1182,7 @@ public class NNThroughputBenchmark implements Tool {
throws IOException {
ExtendedBlock prevBlock = null;
for(int jdx = 0; jdx < blocksPerFile; jdx++) {
- LocatedBlock loc = clientProto.addBlock(fileName, clientName,
+ LocatedBlock loc = addBlock(fileName, clientName,
prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null);
prevBlock = loc.getBlock();
for(DatanodeInfo dnInfo : loc.getLocations()) {
@@ -1195,11 +1196,42 @@ public class NNThroughputBenchmark implements Tool {
dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration,
bpid, report);
}
+ // IBRs are asynchronously processed by NameNode. The next
+ // ClientProtocol#addBlock() may throw NotReplicatedYetException.
}
return prevBlock;
}
/**
+ * Retry ClientProtocol.addBlock() if it throws NotReplicatedYetException.
+ * Because addBlock() also commits the previous block,
+ * it fails if enough IBRs are not processed by NameNode.
+ */
+ private LocatedBlock addBlock(String src, String clientName,
+ ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId,
+ String[] favoredNodes) throws IOException {
+ for (int i = 0; i < 30; i++) {
+ try {
+ return clientProto.addBlock(src, clientName,
+ previous, excludeNodes, fileId, favoredNodes);
+ } catch (NotReplicatedYetException|RemoteException e) {
+ if (e instanceof RemoteException) {
+ String className = ((RemoteException) e).getClassName();
+ if (!className.equals(NotReplicatedYetException.class.getName())) {
+ throw e;
+ }
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ie) {
+ LOG.warn("interrupted while retrying addBlock.", ie);
+ }
+ }
+ }
+ throw new IOException("failed to add block.");
+ }
+
+ /**
* Does not require the argument
*/
@Override
[06/50] [abbrv] hadoop git commit: HDFS-9576: HTrace: collect
position/length information on read operations (zhz via cmccabe)
Posted by ka...@apache.org.
HDFS-9576: HTrace: collect position/length information on read operations (zhz via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7905788d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7905788d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7905788d
Branch: refs/heads/YARN-1011
Commit: 7905788db94d560e6668af0d4bed22b326961aaf
Parents: 1425578
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Jan 20 11:26:44 2016 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Jan 20 11:26:44 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 19 +++++++++++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 34 ++++++++++++++------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
3 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7905788d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 2c42cbe..3b9210a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2967,6 +2967,25 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/**
+ * Full detailed tracing for read requests: path, position in the file,
+ * and length.
+ *
+ * @param reqLen requested length
+ */
+ TraceScope newReaderTraceScope(String description, String path, long pos,
+ int reqLen) {
+ TraceScope scope = newPathTraceScope(description, path);
+ scope.addKVAnnotation("pos", Long.toString(pos));
+ scope.addKVAnnotation("reqLen", Integer.toString(reqLen));
+ return scope;
+ }
+
+ /** Add the returned length info to the scope. */
+ void addRetLenToReaderScope(TraceScope scope, int retLen) {
+ scope.addKVAnnotation("retLen", Integer.toString(retLen));
+ }
+
+ /**
* Get the erasure coding policy information for the specified path
*
* @param src path to get the information for
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7905788d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index b6b11ee..3de60b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -973,18 +973,29 @@ public class DFSInputStream extends FSInputStream
public synchronized int read(@Nonnull final byte buf[], int off, int len)
throws IOException {
ReaderStrategy byteArrayReader = new ByteArrayStrategy(buf);
- try (TraceScope ignored =
- dfsClient.newPathTraceScope("DFSInputStream#byteArrayRead", src)) {
- return readWithStrategy(byteArrayReader, off, len);
+ try (TraceScope scope =
+ dfsClient.newReaderTraceScope("DFSInputStream#byteArrayRead",
+ src, getPos(), len)) {
+ int retLen = readWithStrategy(byteArrayReader, off, len);
+ if (retLen < len) {
+ dfsClient.addRetLenToReaderScope(scope, retLen);
+ }
+ return retLen;
}
}
@Override
public synchronized int read(final ByteBuffer buf) throws IOException {
ReaderStrategy byteBufferReader = new ByteBufferStrategy(buf);
- try (TraceScope ignored =
- dfsClient.newPathTraceScope("DFSInputStream#byteBufferRead", src)){
- return readWithStrategy(byteBufferReader, 0, buf.remaining());
+ int reqLen = buf.remaining();
+ try (TraceScope scope =
+ dfsClient.newReaderTraceScope("DFSInputStream#byteBufferRead",
+ src, getPos(), reqLen)){
+ int retLen = readWithStrategy(byteBufferReader, 0, reqLen);
+ if (retLen < reqLen) {
+ dfsClient.addRetLenToReaderScope(scope, retLen);
+ }
+ return retLen;
}
}
@@ -1433,9 +1444,14 @@ public class DFSInputStream extends FSInputStream
@Override
public int read(long position, byte[] buffer, int offset, int length)
throws IOException {
- try (TraceScope ignored = dfsClient.
- newPathTraceScope("DFSInputStream#byteArrayPread", src)) {
- return pread(position, buffer, offset, length);
+ try (TraceScope scope = dfsClient.
+ newReaderTraceScope("DFSInputStream#byteArrayPread",
+ src, position, length)) {
+ int retLen = pread(position, buffer, offset, length);
+ if (retLen < length) {
+ dfsClient.addRetLenToReaderScope(scope, retLen);
+ }
+ return retLen;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7905788d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e6074bd..7a5290e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -936,6 +936,9 @@ Release 2.9.0 - UNRELEASED
HDFS-9624. DataNode start slowly due to the initial DU command operations.
(Lin Yiqun via wang)
+ HDFS-9576: HTrace: collect position/length information on read operations
+ (zhz via cmccabe)
+
OPTIMIZATIONS
BUG FIXES