You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/02/22 20:47:20 UTC
[01/50] [abbrv] hadoop git commit: Set the release date for 2.6.4.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-1312 a0efd5328 -> a140b213b
Set the release date for 2.6.4.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec12ce8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec12ce8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec12ce8f
Branch: refs/heads/HDFS-1312
Commit: ec12ce8f48944c38271399a6e452aa2c2a9f5b80
Parents: 36ba5ca
Author: Junping Du <ju...@apache.org>
Authored: Sat Feb 13 01:42:50 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Sat Feb 13 01:42:50 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec12ce8f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2ddf0dd..a05fd4b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -2557,7 +2557,7 @@ Release 2.6.5 - UNRELEASED
HADOOP-12589. Fix intermittent test failure of TestCopyPreserveFlag
(iwasakims)
-Release 2.6.4 - UNRELEASED
+Release 2.6.4 - 2016-02-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec12ce8f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8fceee2..eeff967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -3968,7 +3968,7 @@ Release 2.6.5 - UNRELEASED
HDFS-9347. Invariant assumption in TestQuorumJournalManager.shutdown()
is wrong. (Wei-Chiu Chuang via zhz)
-Release 2.6.4 - UNRELEASED
+Release 2.6.4 - 2016-02-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec12ce8f/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 7bcf629..0b8c818 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1044,7 +1044,7 @@ Release 2.6.5 - UNRELEASED
MAPREDUCE-6191. Improve clearing stale state of Java serialization
testcase. (Sam Liu via Eric Yang)
-Release 2.6.4 - UNRELEASED
+Release 2.6.4 - 2016-02-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec12ce8f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 279508e..1489976 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -2353,7 +2353,7 @@ Release 2.6.5 - UNRELEASED
BUG FIXES
-Release 2.6.4 - UNRELEASED
+Release 2.6.4 - 2016-02-11
INCOMPATIBLE CHANGES
[31/50] [abbrv] hadoop git commit: MAPREDUCE-6627. Add
machine-readable output to mapred job -history command (rkanter)
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
new file mode 100644
index 0000000..358e9b2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
@@ -0,0 +1,945 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.junit.Assert;
+import org.junit.Test;
+import org.skyscreamer.jsonassert.JSONAssert;
+import org.skyscreamer.jsonassert.JSONCompareMode;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.TimeZone;
+
+public class TestHistoryViewerPrinter {
+
+ private static final Log LOG = LogFactory.getLog(
+ TestHistoryViewerPrinter.class);
+
+ @Test
+ public void testHumanPrinter() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ HumanReadableHistoryViewerPrinter printer =
+ new HumanReadableHistoryViewerPrinter(job, false, "http://",
+ TimeZone.getTimeZone("GMT"));
+ String outStr = run(printer);
+ Assert.assertEquals("\n" +
+ "Hadoop job: job_1317928501754_0001\n" +
+ "=====================================\n" +
+ "User: rkanter\n" +
+ "JobName: my job\n" +
+ "JobConf: /tmp/job.xml\n" +
+ "Submitted At: 6-Oct-2011 19:15:01\n" +
+ "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" +
+ "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" +
+ "Status: SUCCEEDED\n" +
+ "Counters: \n" +
+ "\n" +
+ "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" +
+ "---------------------------------------------------------------------------------------\n" +
+ "|group1 |counter1 |5 |5 |5 \n" +
+ "|group1 |counter2 |10 |10 |10 \n" +
+ "|group2 |counter1 |15 |15 |15 \n" +
+ "\n" +
+ "=====================================\n" +
+ "\n" +
+ "Task Summary\n" +
+ "============================\n" +
+ "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" +
+ "\n" +
+ "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" +
+ "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" +
+ "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" +
+ "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" +
+ "============================\n" +
+ "\n" +
+ "\n" +
+ "Analysis\n" +
+ "=========\n" +
+ "\n" +
+ "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" +
+ "Average time taken by map tasks: 5sec\n" +
+ "Worse performing map tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_m_000007 7sec\n" +
+ "task_1317928501754_0001_m_000006 6sec\n" +
+ "task_1317928501754_0001_m_000005 5sec\n" +
+ "task_1317928501754_0001_m_000004 4sec\n" +
+ "task_1317928501754_0001_m_000003 3sec\n" +
+ "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" +
+ "\n" +
+ "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" +
+ "Average time taken by shuffle tasks: 8sec\n" +
+ "Worse performing shuffle tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 8sec\n" +
+ "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "\n" +
+ "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" +
+ "Average time taken by reduce tasks: 0sec\n" +
+ "Worse performing reduce tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 0sec\n" +
+ "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "=========\n" +
+ "\n" +
+ "FAILED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" +
+ "\n" +
+ "FAILED task attempts by nodes\n" +
+ "Hostname\tFailedTasks\n" +
+ "===============================\n" +
+ "localhost\ttask_1317928501754_0001_m_000002, \n", outStr);
+ }
+
+ @Test
+ public void testHumanPrinterAll() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ HumanReadableHistoryViewerPrinter printer =
+ new HumanReadableHistoryViewerPrinter(job, true, "http://",
+ TimeZone.getTimeZone("GMT"));
+ String outStr = run(printer);
+ if (System.getProperty("java.version").startsWith("1.7")) {
+ Assert.assertEquals("\n" +
+ "Hadoop job: job_1317928501754_0001\n" +
+ "=====================================\n" +
+ "User: rkanter\n" +
+ "JobName: my job\n" +
+ "JobConf: /tmp/job.xml\n" +
+ "Submitted At: 6-Oct-2011 19:15:01\n" +
+ "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" +
+ "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" +
+ "Status: SUCCEEDED\n" +
+ "Counters: \n" +
+ "\n" +
+ "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" +
+ "---------------------------------------------------------------------------------------\n" +
+ "|group1 |counter1 |5 |5 |5 \n" +
+ "|group1 |counter2 |10 |10 |10 \n" +
+ "|group2 |counter1 |15 |15 |15 \n" +
+ "\n" +
+ "=====================================\n" +
+ "\n" +
+ "Task Summary\n" +
+ "============================\n" +
+ "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" +
+ "\n" +
+ "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" +
+ "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" +
+ "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" +
+ "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" +
+ "============================\n" +
+ "\n" +
+ "\n" +
+ "Analysis\n" +
+ "=========\n" +
+ "\n" +
+ "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" +
+ "Average time taken by map tasks: 5sec\n" +
+ "Worse performing map tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_m_000007 7sec\n" +
+ "task_1317928501754_0001_m_000006 6sec\n" +
+ "task_1317928501754_0001_m_000005 5sec\n" +
+ "task_1317928501754_0001_m_000004 4sec\n" +
+ "task_1317928501754_0001_m_000003 3sec\n" +
+ "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" +
+ "\n" +
+ "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" +
+ "Average time taken by shuffle tasks: 8sec\n" +
+ "Worse performing shuffle tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 8sec\n" +
+ "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "\n" +
+ "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" +
+ "Average time taken by reduce tasks: 0sec\n" +
+ "Worse performing reduce tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 0sec\n" +
+ "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "=========\n" +
+ "\n" +
+ "FAILED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_s_000001\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000006\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000005\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000004\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000003\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000007\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_r_000008\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_c_000009\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\t\n" +
+ "\n" +
+ "JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_s_000001_1\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\n" +
+ "\n" +
+ "MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_m_000002_1\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\n" +
+ "attempt_1317928501754_0001_m_000006_1\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\n" +
+ "attempt_1317928501754_0001_m_000005_1\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\n" +
+ "attempt_1317928501754_0001_m_000004_1\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\n" +
+ "attempt_1317928501754_0001_m_000003_1\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\n" +
+ "attempt_1317928501754_0001_m_000007_1\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\n" +
+ "\n" +
+ "REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tShuffleFinished\tSortFinished\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_r_000008_1\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t6-Oct-2011 19:15:18 (0sec)6-Oct-2011 19:15:18 (8sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\n" +
+ "\n" +
+ "JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_c_000009_1\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\n" +
+ "\n" +
+ "FAILED task attempts by nodes\n" +
+ "Hostname\tFailedTasks\n" +
+ "===============================\n" +
+ "localhost\ttask_1317928501754_0001_m_000002, \n", outStr);
+ } else {
+ Assert.assertEquals("\n" +
+ "Hadoop job: job_1317928501754_0001\n" +
+ "=====================================\n" +
+ "User: rkanter\n" +
+ "JobName: my job\n" +
+ "JobConf: /tmp/job.xml\n" +
+ "Submitted At: 6-Oct-2011 19:15:01\n" +
+ "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" +
+ "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" +
+ "Status: SUCCEEDED\n" +
+ "Counters: \n" +
+ "\n" +
+ "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" +
+ "---------------------------------------------------------------------------------------\n" +
+ "|group1 |counter1 |5 |5 |5 \n" +
+ "|group1 |counter2 |10 |10 |10 \n" +
+ "|group2 |counter1 |15 |15 |15 \n" +
+ "\n" +
+ "=====================================\n" +
+ "\n" +
+ "Task Summary\n" +
+ "============================\n" +
+ "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" +
+ "\n" +
+ "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" +
+ "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" +
+ "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" +
+ "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" +
+ "============================\n" +
+ "\n" +
+ "\n" +
+ "Analysis\n" +
+ "=========\n" +
+ "\n" +
+ "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" +
+ "Average time taken by map tasks: 5sec\n" +
+ "Worse performing map tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_m_000007 7sec\n" +
+ "task_1317928501754_0001_m_000006 6sec\n" +
+ "task_1317928501754_0001_m_000005 5sec\n" +
+ "task_1317928501754_0001_m_000004 4sec\n" +
+ "task_1317928501754_0001_m_000003 3sec\n" +
+ "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" +
+ "\n" +
+ "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" +
+ "Average time taken by shuffle tasks: 8sec\n" +
+ "Worse performing shuffle tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 8sec\n" +
+ "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "\n" +
+ "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" +
+ "Average time taken by reduce tasks: 0sec\n" +
+ "Worse performing reduce tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 0sec\n" +
+ "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "=========\n" +
+ "\n" +
+ "FAILED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_s_000001\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000007\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000006\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000005\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000004\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000003\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_r_000008\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_c_000009\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\t\n" +
+ "\n" +
+ "JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_s_000001_1\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\n" +
+ "\n" +
+ "MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_m_000007_1\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\n" +
+ "attempt_1317928501754_0001_m_000002_1\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\n" +
+ "attempt_1317928501754_0001_m_000006_1\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\n" +
+ "attempt_1317928501754_0001_m_000005_1\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\n" +
+ "attempt_1317928501754_0001_m_000004_1\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\n" +
+ "attempt_1317928501754_0001_m_000003_1\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\n" +
+ "\n" +
+ "REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tShuffleFinished\tSortFinished\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_r_000008_1\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t6-Oct-2011 19:15:18 (0sec)6-Oct-2011 19:15:18 (8sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\n" +
+ "\n" +
+ "JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_c_000009_1\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\n" +
+ "\n" +
+ "FAILED task attempts by nodes\n" +
+ "Hostname\tFailedTasks\n" +
+ "===============================\n" +
+ "localhost\ttask_1317928501754_0001_m_000002, \n", outStr);
+ }
+ }
+
+ @Test
+ public void testJSONPrinter() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ JSONHistoryViewerPrinter printer =
+ new JSONHistoryViewerPrinter(job, false, "http://");
+ String outStr = run(printer);
+ JSONAssert.assertEquals("{\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 5,\n" +
+ " \"reduceValue\": 5,\n" +
+ " \"totalValue\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"mapValue\": 10,\n" +
+ " \"reduceValue\": 10,\n" +
+ " \"totalValue\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 15,\n" +
+ " \"reduceValue\": 15,\n" +
+ " \"totalValue\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishedAt\": 1317928516754,\n" +
+ " \"hadoopJob\": \"job_1317928501754_0001\",\n" +
+ " \"jobConf\": \"/tmp/job.xml\",\n" +
+ " \"jobName\": \"my job\",\n" +
+ " \"launchedAt\": 1317928502754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"submittedAt\": 1317928501754,\n" +
+ " \"taskSummary\": {\n" +
+ " \"cleanup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"map\": {\n" +
+ " \"failed\": 1,\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"successful\": 5,\n" +
+ " \"total\": 6\n" +
+ " },\n" +
+ " \"reduce\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"setup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " }\n" +
+ " },\n" +
+ " \"tasks\": [\n" +
+ " {\n" +
+ " \"finishTime\": 1317928506754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"status\": \"FAILED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000002\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"user\": \"rkanter\"\n" +
+ "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
+ }
+
+ @Test
+ public void testJSONPrinterAll() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ JSONHistoryViewerPrinter printer =
+ new JSONHistoryViewerPrinter(job, true, "http://");
+ String outStr = run(printer);
+ JSONAssert.assertEquals("{\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 5,\n" +
+ " \"reduceValue\": 5,\n" +
+ " \"totalValue\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"mapValue\": 10,\n" +
+ " \"reduceValue\": 10,\n" +
+ " \"totalValue\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 15,\n" +
+ " \"reduceValue\": 15,\n" +
+ " \"totalValue\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishedAt\": 1317928516754,\n" +
+ " \"hadoopJob\": \"job_1317928501754_0001\",\n" +
+ " \"jobConf\": \"/tmp/job.xml\",\n" +
+ " \"jobName\": \"my job\",\n" +
+ " \"launchedAt\": 1317928502754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"submittedAt\": 1317928501754,\n" +
+ " \"taskSummary\": {\n" +
+ " \"cleanup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"map\": {\n" +
+ " \"failed\": 1,\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"successful\": 5,\n" +
+ " \"total\": 6\n" +
+ " },\n" +
+ " \"reduce\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"setup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " }\n" +
+ " },\n" +
+ " \"tasks\": [\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000002_1\",\n" +
+ " \"finishTime\": 1317928506754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928506754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"status\": \"FAILED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000002\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_s_000001_1\",\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_s_000001\",\n" +
+ " \"type\": \"JOB_SETUP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000006_1\",\n" +
+ " \"finishTime\": 1317928514754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928508754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928514754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928508754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000006\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000005_1\",\n" +
+ " \"finishTime\": 1317928512754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928507754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928512754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928507754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000005\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000004_1\",\n" +
+ " \"finishTime\": 1317928510754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928506754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928510754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928506754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000004\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000003_1\",\n" +
+ " \"finishTime\": 1317928508754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928505754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928508754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928505754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000003\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_c_000009_1\",\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_c_000009\",\n" +
+ " \"type\": \"JOB_CLEANUP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000007_1\",\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928509754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928509754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000007\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_r_000008_1\",\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"shuffleFinished\": 1317928518754,\n" +
+ " \"sortFinished\": 1317928518754,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_r_000008\",\n" +
+ " \"type\": \"REDUCE\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"user\": \"rkanter\"\n" +
+ "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
+ }
+
+ private String run(HistoryViewerPrinter printer) throws Exception {
+ ByteArrayOutputStream boas = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(boas, true);
+ printer.print(out);
+ out.close();
+ String outStr = boas.toString("UTF-8");
+ LOG.info("out = " + outStr);
+ return outStr;
+ }
+
+ private static JobHistoryParser.JobInfo createJobInfo() {
+ JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo();
+ job.submitTime = 1317928501754L;
+ job.finishTime = job.submitTime + 15000;
+ job.jobid = JobID.forName("job_1317928501754_0001");
+ job.username = "rkanter";
+ job.jobname = "my job";
+ job.jobQueueName = "my queue";
+ job.jobConfPath = "/tmp/job.xml";
+ job.launchTime = job.submitTime + 1000;
+ job.totalMaps = 5;
+ job.totalReduces = 1;
+ job.failedMaps = 1;
+ job.failedReduces = 0;
+ job.finishedMaps = 5;
+ job.finishedReduces = 1;
+ job.jobStatus = JobStatus.State.SUCCEEDED.name();
+ job.totalCounters = createCounters();
+ job.mapCounters = createCounters();
+ job.reduceCounters = createCounters();
+ job.tasksMap = new HashMap<>();
+ addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.FAILED);
+ addTaskInfo(job, TaskType.MAP, 3, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 4, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 5, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 6, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 7, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.REDUCE, 8, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED);
+ return job;
+ }
+
+ private static Counters createCounters() {
+ Counters counters = new Counters();
+ counters.findCounter("group1", "counter1").setValue(5);
+ counters.findCounter("group1", "counter2").setValue(10);
+ counters.findCounter("group2", "counter1").setValue(15);
+ return counters;
+ }
+
+ private static void addTaskInfo(JobHistoryParser.JobInfo job,
+ TaskType type, int id, TaskStatus.State status) {
+ JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();
+ task.taskId = new TaskID(job.getJobId(), type, id);
+ task.startTime = job.getLaunchTime() + id * 1000;
+ task.finishTime = task.startTime + id * 1000;
+ task.taskType = type;
+ task.counters = createCounters();
+ task.status = status.name();
+ task.attemptsMap = new HashMap<>();
+ addTaskAttemptInfo(task, 1);
+ job.tasksMap.put(task.getTaskId(), task);
+ }
+
+ private static void addTaskAttemptInfo(
+ JobHistoryParser.TaskInfo task, int id) {
+ JobHistoryParser.TaskAttemptInfo attempt =
+ new JobHistoryParser.TaskAttemptInfo();
+ attempt.attemptId = new TaskAttemptID(
+ TaskID.downgrade(task.getTaskId()), id);
+ attempt.startTime = task.getStartTime();
+ attempt.finishTime = task.getFinishTime();
+ attempt.shuffleFinishTime = task.getFinishTime();
+ attempt.sortFinishTime = task.getFinishTime();
+ attempt.mapFinishTime = task.getFinishTime();
+ attempt.status = task.getTaskStatus();
+ attempt.taskType = task.getTaskType();
+ attempt.trackerName = "localhost";
+ attempt.httpPort = 1234;
+ attempt.hostname = "localhost";
+ task.attemptsMap.put(attempt.getAttemptId(), attempt);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
index 2382239..f13c163 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
@@ -29,6 +30,8 @@ import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -359,27 +362,152 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
String historyFileUri = new Path(f.getAbsolutePath())
.makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toUri()
.toString();
-
- // bad command
- int exitCode = runTool(conf, jc, new String[] { "-history", "pul",
- historyFileUri }, out);
- assertEquals("Exit code", -1, exitCode);
- exitCode = runTool(conf, jc, new String[] { "-history", "all",
- historyFileUri }, out);
+ // Try a bunch of different valid combinations of the command to test
+ // argument parsing
+ int exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ }, out);
assertEquals("Exit code", 0, exitCode);
- String line;
+ checkHistoryHumanOutput(out);
+ File outFile = File.createTempFile("myout", ".txt");
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-outfile",
+ outFile.getAbsolutePath()
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryHumanFileOutput(out, outFile);
+ outFile = File.createTempFile("myout", ".txt");
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-outfile",
+ outFile.getAbsolutePath(),
+ "-format",
+ "human"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryHumanFileOutput(out, outFile);
+
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-format",
+ "human"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryHumanOutput(out);
+
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-format",
+ "json"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryJSONOutput(out);
+ outFile = File.createTempFile("myout", ".txt");
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-outfile",
+ outFile.getAbsolutePath(),
+ "-format",
+ "json"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryJSONFileOutput(out, outFile);
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-format",
+ "json"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryJSONOutput(out);
+
+ // Check some bad arguments
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "foo"
+ }, out);
+ assertEquals("Exit code", -1, exitCode);
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-format"
+ }, out);
+ assertEquals("Exit code", -1, exitCode);
+ runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-outfile",
+ }, out);
+ assertEquals("Exit code", -1, exitCode);
+ try {
+ runTool(conf, jc, new String[]{
+ "-history",
+ historyFileUri,
+ "-format",
+ "foo"
+ }, out);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+ }
+
+ private void checkHistoryHumanOutput(ByteArrayOutputStream out)
+ throws IOException, JSONException {
BufferedReader br = new BufferedReader(new InputStreamReader(
new ByteArrayInputStream(out.toByteArray())));
- int counter = 0;
- while ((line = br.readLine()) != null) {
- LOG.info("line = " + line);
- if (line.startsWith("task_")) {
- counter++;
- }
- }
- assertEquals(23, counter);
+ br.readLine();
+ String line = br.readLine();
+ br.close();
+ assertEquals("Hadoop job: job_1329348432655_0001", line);
+ out.reset();
+ }
+
+ private void checkHistoryJSONOutput(ByteArrayOutputStream out)
+ throws IOException, JSONException {
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ new ByteArrayInputStream(out.toByteArray())));
+ String line = org.apache.commons.io.IOUtils.toString(br);
+ br.close();
+ JSONObject json = new JSONObject(line);
+ assertEquals("job_1329348432655_0001", json.getString("hadoopJob"));
+ out.reset();
+ }
+
+ private void checkHistoryHumanFileOutput(ByteArrayOutputStream out,
+ File outFile) throws IOException, JSONException {
+ BufferedReader br = new BufferedReader(new FileReader(outFile));
+ br.readLine();
+ String line = br.readLine();
+ br.close();
+ assertEquals("Hadoop job: job_1329348432655_0001", line);
+ assertEquals(0, out.size());
+ }
+
+ private void checkHistoryJSONFileOutput(ByteArrayOutputStream out,
+ File outFile) throws IOException, JSONException {
+ BufferedReader br = new BufferedReader(new FileReader(outFile));
+ String line = org.apache.commons.io.IOUtils.toString(br);
+ br.close();
+ JSONObject json = new JSONObject(line);
+ assertEquals("job_1329348432655_0001", json.getString("hadoopJob"));
+ assertEquals(0, out.size());
}
+
/**
* print job events list
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 6078573..f23b46e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -992,6 +992,12 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.skyscreamer</groupId>
+ <artifactId>jsonassert</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
[35/50] [abbrv] hadoop git commit: HADOOP-12828. Print user when
services are started. (Wei-Chiu Chuang via Yongjun Zhang)
Posted by ar...@apache.org.
HADOOP-12828. Print user when services are started. (Wei-Chiu Chuang via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a963baba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a963baba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a963baba
Branch: refs/heads/HDFS-1312
Commit: a963baba10648859ad88d4ed9a4325b66b1af0d5
Parents: 748b6c0
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Fri Feb 19 09:41:22 2016 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Fri Feb 19 09:41:22 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/StringUtils.java | 1 +
.../java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java | 1 +
3 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a963baba/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 91ef68e..bf84d7a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1134,6 +1134,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11031. Design Document for Credential Provider API.
(Larry McCay via cnauroth)
+ HADOOP-12828. Print user when services are started.
+ (Wei-Chiu Chuang via Yongjun Zhang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a963baba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 1ccdb22..31f6914 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -687,6 +687,7 @@ public class StringUtils {
LOG.info(
toStartupShutdownString("STARTUP_MSG: ", new String[] {
"Starting " + classname,
+ " user = " + System.getProperty("user.name"),
" host = " + hostname,
" args = " + Arrays.asList(args),
" version = " + VersionInfo.getVersion(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a963baba/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index 325f8db..3b439a4 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -124,6 +124,7 @@ public class KMSWebApp implements ServletContextListener {
LOG.info("-------------------------------------------------------------");
LOG.info(" Java runtime version : {}", System.getProperty(
"java.runtime.version"));
+ LOG.info(" User: {}", System.getProperty("user.name"));
LOG.info(" KMS Hadoop Version: " + VersionInfo.getVersion());
LOG.info("-------------------------------------------------------------");
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a140b213
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a140b213
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a140b213
Branch: refs/heads/HDFS-1312
Commit: a140b213bfe4f975e2f9184753d06d7b7c794168
Parents: 3f72fcc a63301a
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Feb 22 11:46:03 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Feb 22 11:46:03 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 110 +-
.../src/main/bin/hadoop-functions.sh | 1 +
.../hadoop-common/src/main/conf/hadoop-env.sh | 6 +
.../org/apache/hadoop/conf/Reconfigurable.java | 8 +-
.../apache/hadoop/conf/ReconfigurableBase.java | 47 +-
.../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 3 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 14 +-
.../java/org/apache/hadoop/fs/CanUnbuffer.java | 2 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 45 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +-
.../java/org/apache/hadoop/fs/TrashPolicy.java | 2 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 9 +-
.../org/apache/hadoop/fs/shell/AclCommands.java | 12 +-
.../org/apache/hadoop/fs/shell/FsCommand.java | 2 +-
.../hadoop/io/compress/zlib/ZlibCompressor.java | 42 +-
.../rawcoder/AbstractRawErasureDecoder.java | 39 +-
.../io/erasurecode/rawcoder/RSRawDecoder.java | 11 +-
.../io/erasurecode/rawcoder/RSRawDecoder2.java | 176 +++
.../io/erasurecode/rawcoder/RSRawEncoder2.java | 76 ++
.../rawcoder/RSRawErasureCoderFactory2.java | 37 +
.../erasurecode/rawcoder/RawErasureCoder.java | 2 +-
.../io/erasurecode/rawcoder/util/CoderUtil.java | 83 ++
.../io/erasurecode/rawcoder/util/DumpUtil.java | 18 +-
.../io/erasurecode/rawcoder/util/GF256.java | 339 +++++
.../io/erasurecode/rawcoder/util/RSUtil.java | 2 +-
.../io/erasurecode/rawcoder/util/RSUtil2.java | 172 +++
.../metrics2/sink/RollingFileSystemSink.java | 409 ++++--
.../hadoop/metrics2/source/JvmMetrics.java | 2 +-
.../net/NetworkTopologyWithNodeGroup.java | 7 +-
.../java/org/apache/hadoop/net/NodeBase.java | 9 +-
.../java/org/apache/hadoop/security/KDiag.java | 976 +++++++++++++
.../hadoop/security/UserGroupInformation.java | 36 +-
.../security/http/RestCsrfPreventionFilter.java | 218 ++-
.../apache/hadoop/security/ssl/SSLFactory.java | 3 +-
.../org/apache/hadoop/util/DataChecksum.java | 147 +-
.../org/apache/hadoop/util/JvmPauseMonitor.java | 2 +-
.../org/apache/hadoop/util/StringUtils.java | 1 +
.../hadoop/util/concurrent/ExecutorHelper.java | 67 +
.../hadoop/util/concurrent/HadoopExecutors.java | 96 ++
.../HadoopScheduledThreadPoolExecutor.java | 71 +
.../concurrent/HadoopThreadPoolExecutor.java | 92 ++
.../hadoop/util/concurrent/package-info.java | 26 +
.../src/main/resources/core-default.xml | 15 +-
...he.hadoop.application-classloader.properties | 1 +
.../src/site/markdown/CommandsManual.md | 14 +-
.../src/site/markdown/CredentialProviderAPI.md | 132 ++
.../src/site/markdown/FileSystemShell.md | 2 +-
.../hadoop-common/src/site/markdown/Metrics.md | 27 +-
.../src/site/markdown/SecureMode.md | 198 +++
.../apache/hadoop/conf/TestReconfiguration.java | 143 +-
.../fs/contract/localfs/LocalFSContract.java | 6 +-
.../hadoop/fs/shell/TestCopyPreserveFlag.java | 5 +-
.../java/org/apache/hadoop/fs/shell/TestLs.java | 2 +-
.../apache/hadoop/http/TestHttpServerLogs.java | 2 +-
.../zlib/TestZlibCompressorDecompressor.java | 25 +
.../hadoop/io/erasurecode/TestECSchema.java | 6 +
.../erasurecode/coder/TestRSErasureCoder.java | 4 +
.../io/erasurecode/coder/TestXORCoder.java | 5 +
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 91 --
.../erasurecode/rawcoder/TestRSRawCoder2.java | 33 +
.../rawcoder/TestRSRawCoderBase.java | 109 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 1 -
.../sink/RollingFileSystemSinkTestBase.java | 87 +-
.../net/TestNetworkTopologyWithNodeGroup.java | 15 +-
.../org/apache/hadoop/security/TestKDiag.java | 226 +++
.../apache/hadoop/security/TestKDiagNoKDC.java | 123 ++
.../security/TestUserGroupInformation.java | 2 +-
.../http/TestRestCsrfPreventionFilter.java | 93 +-
.../hadoop/util/Crc32PerformanceTest.java | 350 +++++
.../apache/hadoop/util/TestDataChecksum.java | 6 +
.../apache/hadoop/security/secure-hdfs-site.xml | 26 +
.../hadoop-kms/src/main/conf/kms-env.sh | 4 +
...rKeyGeneratorKeyProviderCryptoExtension.java | 10 +
.../key/kms/server/KMSExceptionsProvider.java | 2 +
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 1 +
.../hadoop-kms/src/main/libexec/kms-config.sh | 1 +
.../hadoop-kms/src/main/tomcat/server.xml | 3 +-
.../src/main/tomcat/ssl-server.xml.conf | 1 +
.../hadoop-kms/src/site/markdown/index.md.vm | 78 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 21 +-
.../main/java/org/apache/hadoop/fs/Hdfs.java | 535 ++++++++
.../java/org/apache/hadoop/fs/SWebHdfs.java | 64 +
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 63 +
.../main/java/org/apache/hadoop/fs/package.html | 26 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 1 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 28 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 7 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 132 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 43 +-
.../hadoop/hdfs/DistributedFileSystem.java | 67 +-
.../hdfs/client/HdfsClientConfigKeys.java | 16 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 13 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 2 +-
.../hdfs/server/protocol/DatanodeStorage.java | 1 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 65 +-
...onfRefreshTokenBasedAccessTokenProvider.java | 8 +-
.../CredentialBasedAccessTokenProvider.java | 8 +-
.../src/main/proto/erasurecoding.proto | 4 +-
.../apache/hadoop/lib/servlet/ServerWebApp.java | 23 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 220 ++-
.../jdiff/Apache_Hadoop_HDFS_2.6.0.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +
.../main/java/org/apache/hadoop/fs/Hdfs.java | 535 --------
.../java/org/apache/hadoop/fs/SWebHdfs.java | 64 -
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 63 -
.../main/java/org/apache/hadoop/fs/package.html | 26 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 33 +-
.../java/org/apache/hadoop/hdfs/HAUtil.java | 27 +
.../apache/hadoop/hdfs/net/TcpPeerServer.java | 6 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 5 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 95 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 11 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 202 +--
.../blockmanagement/BlockInfoContiguous.java | 29 +-
.../blockmanagement/BlockInfoStriped.java | 30 +-
.../server/blockmanagement/BlockManager.java | 667 +++++----
.../BlockPlacementPolicyDefault.java | 12 +-
.../BlockPlacementPolicyRackFaultTolerant.java | 10 +-
.../BlockPlacementPolicyWithNodeGroup.java | 46 +
.../BlockPlacementStatusDefault.java | 10 +-
.../BlockPlacementStatusWithNodeGroup.java | 81 ++
.../BlockReconstructionWork.java | 116 ++
.../blockmanagement/BlockRecoveryWork.java | 111 --
.../BlockUnderConstructionFeature.java | 25 +
.../hdfs/server/blockmanagement/BlocksMap.java | 88 +-
.../blockmanagement/DatanodeDescriptor.java | 13 +-
.../server/blockmanagement/DatanodeManager.java | 12 +-
.../blockmanagement/DatanodeStorageInfo.java | 123 +-
.../blockmanagement/ErasureCodingWork.java | 111 +-
.../PendingReplicationBlocks.java | 9 +-
.../server/blockmanagement/ReplicationWork.java | 11 +-
.../hdfs/server/datanode/BPOfferService.java | 54 +-
.../hdfs/server/datanode/BPServiceActor.java | 275 +---
.../hadoop/hdfs/server/datanode/DataNode.java | 123 +-
.../hdfs/server/datanode/DataStorage.java | 47 +-
.../hdfs/server/datanode/DataXceiver.java | 1 +
.../hdfs/server/datanode/DataXceiverServer.java | 2 +-
.../datanode/IncrementalBlockReportManager.java | 224 +++
.../server/datanode/SecureDataNodeStarter.java | 6 +-
.../hdfs/server/datanode/VolumeScanner.java | 7 +-
.../erasurecode/ErasureCodingWorker.java | 247 ++--
.../AvailableSpaceVolumeChoosingPolicy.java | 35 +-
.../RoundRobinVolumeChoosingPolicy.java | 53 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 29 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 60 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 69 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 47 +-
.../datanode/fsdataset/impl/ReplicaMap.java | 71 +-
.../server/datanode/web/DatanodeHttpServer.java | 114 +-
.../web/PortUnificationServerHandler.java | 18 +-
.../web/RestCsrfPreventionFilterHandler.java | 137 ++
.../server/namenode/EncryptionZoneManager.java | 73 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 8 +-
.../hdfs/server/namenode/FSEditLogOp.java | 41 +-
.../hdfs/server/namenode/FSNamesystem.java | 26 +-
.../hadoop/hdfs/server/namenode/INode.java | 6 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 40 +-
.../hdfs/server/namenode/INodeReference.java | 5 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 3 +-
.../server/namenode/NameNodeHttpServer.java | 26 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 23 -
.../hdfs/server/namenode/ha/EditLogTailer.java | 6 +-
.../ha/RequestHedgingProxyProvider.java | 41 +-
.../server/namenode/ha/StandbyCheckpointer.java | 7 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 11 +-
.../snapshot/FileWithSnapshotFeature.java | 2 +-
.../top/window/RollingWindowManager.java | 8 +-
.../protocol/BlockECReconstructionCommand.java | 148 ++
.../server/protocol/BlockECRecoveryCommand.java | 147 --
.../server/protocol/BlockReportContext.java | 10 +-
.../hdfs/server/protocol/DatanodeProtocol.java | 3 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../tools/offlineImageViewer/FSImageLoader.java | 12 +-
.../IgnoreSnapshotException.java | 28 +
.../PBImageDelimitedTextWriter.java | 27 +-
.../offlineImageViewer/PBImageTextWriter.java | 90 +-
.../apache/hadoop/hdfs/util/FoldedTreeSet.java | 1285 ++++++++++++++++++
.../org/apache/hadoop/hdfs/web/AuthFilter.java | 10 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 27 +-
.../src/main/proto/DatanodeProtocol.proto | 17 +-
.../src/main/resources/hdfs-default.xml | 89 +-
.../src/main/webapps/datanode/index.html | 2 +-
.../src/main/webapps/hdfs/dfshealth.html | 2 +-
.../src/main/webapps/hdfs/explorer.html | 3 +-
.../src/main/webapps/journal/index.html | 2 +-
.../src/main/webapps/secondary/status.html | 2 +-
.../src/main/webapps/static/rest-csrf.js | 91 ++
.../src/site/markdown/HDFSCommands.md | 2 +
.../src/site/markdown/TransparentEncryption.md | 64 +-
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 8 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 36 +
.../apache/hadoop/cli/TestErasureCodingCLI.java | 6 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +-
.../hdfs/ErasureCodeBenchmarkThroughput.java | 17 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 +
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 13 +-
.../TestClientProtocolForPipelineRecovery.java | 60 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 8 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 35 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 5 +
.../TestDFSStripedOutputStreamWithFailure.java | 119 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 46 +-
.../TestErasureCodeBenchmarkThroughput.java | 5 +
.../TestErasureCodingPolicyWithSnapshot.java | 8 +-
.../org/apache/hadoop/hdfs/TestFileAppend.java | 23 +-
.../org/apache/hadoop/hdfs/TestFileAppend2.java | 8 -
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 5 -
.../hadoop/hdfs/TestFileStatusWithECPolicy.java | 5 +
.../org/apache/hadoop/hdfs/TestLargeBlock.java | 5 -
.../hdfs/TestReadStripedFileWithDecoding.java | 29 +-
.../TestReadStripedFileWithMissingBlocks.java | 55 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 431 ++++++
.../hadoop/hdfs/TestRecoverStripedFile.java | 425 ------
.../apache/hadoop/hdfs/TestRollingUpgrade.java | 4 +-
.../hdfs/TestSafeModeWithStripedFile.java | 9 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 5 +
.../hdfs/TestWriteStripedFileWithFailure.java | 7 +-
.../apache/hadoop/hdfs/UpgradeUtilities.java | 10 +
.../hdfs/protocol/TestBlockListAsLongs.java | 4 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 28 +-
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../server/blockmanagement/TestBlockInfo.java | 88 --
.../blockmanagement/TestBlockInfoStriped.java | 5 +
.../blockmanagement/TestBlockManager.java | 75 +-
.../TestBlockManagerSafeMode.java | 8 +-
.../TestBlockTokenWithDFSStriped.java | 5 +
.../TestBlocksWithNotEnoughRacks.java | 12 +-
.../blockmanagement/TestPendingReplication.java | 9 +-
...constructStripedBlocksWithRackAwareness.java | 213 +++
.../TestReplicationPolicyWithNodeGroup.java | 102 +-
.../server/datanode/SimulatedFSDataset.java | 5 +-
.../TestBlockHasMultipleReplicasOnSameDN.java | 20 +-
.../hdfs/server/datanode/TestBlockScanner.java | 2 +-
.../hdfs/server/datanode/TestDataNodeECN.java | 6 +
.../datanode/TestDataNodeHotSwapVolumes.java | 47 +-
.../server/datanode/TestDataNodeMXBean.java | 55 +
.../datanode/TestDataNodeVolumeFailure.java | 25 +-
.../TestDataNodeVolumeFailureReporting.java | 9 +-
...TestDnRespectsBlockReportSplitThreshold.java | 4 +-
.../datanode/TestIncrementalBlockReports.java | 11 +-
.../TestNNHandlesBlockReportPerStorage.java | 3 +-
.../TestNNHandlesCombinedBlockReport.java | 2 +-
.../server/datanode/TestTriggerBlockReport.java | 12 +-
.../TestRoundRobinVolumeChoosingPolicy.java | 56 +
.../fsdataset/impl/TestFsDatasetImpl.java | 81 ++
.../fsdataset/impl/TestFsVolumeList.java | 42 +-
.../hdfs/server/namenode/FSImageTestUtil.java | 33 +-
.../server/namenode/NNThroughputBenchmark.java | 65 +-
.../TestAddOverReplicatedStripedBlocks.java | 5 +
.../server/namenode/TestAddStripedBlocks.java | 44 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../hdfs/server/namenode/TestFSNamesystem.java | 78 ++
.../hdfs/server/namenode/TestINodeFile.java | 3 +-
.../namenode/TestNNThroughputBenchmark.java | 72 +-
.../TestNameNodeMetadataConsistency.java | 2 +
.../namenode/TestNestedEncryptionZones.java | 215 +++
.../namenode/TestQuotaWithStripedBlocks.java | 5 +
.../namenode/TestReconstructStripedBlocks.java | 239 ++++
.../namenode/TestRecoverStripedBlocks.java | 238 ----
.../server/namenode/TestStripedINodeFile.java | 5 +
.../server/namenode/ha/TestHAConfiguration.java | 23 +
.../namenode/ha/TestStandbyCheckpoints.java | 42 +
.../namenode/snapshot/TestSnapshotDeletion.java | 95 +-
.../shortcircuit/TestShortCircuitLocalRead.java | 73 +-
.../TestOfflineImageViewer.java | 36 +-
.../TestOfflineImageViewerForAcl.java | 36 +
.../hadoop/hdfs/util/FoldedTreeSetTest.java | 644 +++++++++
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 5 +
.../apache/hadoop/hdfs/web/TestAuthFilter.java | 15 +
...TestWebHdfsWithRestCsrfPreventionFilter.java | 166 +++
.../sink/TestRollingFileSystemSinkWithHdfs.java | 286 ++++
...TestRollingFileSystemSinkWithSecureHdfs.java | 283 ++++
.../src/test/resources/testCryptoConf.xml | 8 +-
hadoop-mapreduce-project/CHANGES.txt | 59 +-
.../hadoop/mapred/LocalContainerLauncher.java | 4 +-
.../jobhistory/JobHistoryEventHandler.java | 6 +-
.../v2/app/commit/CommitterEventHandler.java | 3 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 3 +-
.../v2/app/launcher/ContainerLauncherImpl.java | 3 +-
.../mapred/LocalDistributedCacheManager.java | 5 +-
.../apache/hadoop/mapred/LocalJobRunner.java | 8 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 2 +-
.../hadoop-mapreduce-client-core/pom.xml | 5 +
.../org/apache/hadoop/mapred/JobClient.java | 8 +
.../hadoop/mapred/LocatedFileStatusFetcher.java | 4 +-
.../java/org/apache/hadoop/mapred/TaskLog.java | 31 +-
.../mapred/lib/MultithreadedMapRunner.java | 4 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 7 +-
.../mapreduce/jobhistory/HistoryViewer.java | 480 +------
.../jobhistory/HistoryViewerPrinter.java | 39 +
.../HumanReadableHistoryViewerPrinter.java | 471 +++++++
.../jobhistory/JSONHistoryViewerPrinter.java | 256 ++++
.../org/apache/hadoop/mapreduce/tools/CLI.java | 79 +-
.../src/main/resources/mapred-default.xml | 2 +-
.../src/site/markdown/EncryptedShuffle.md | 2 +-
.../src/site/markdown/MapReduceTutorial.md | 6 +-
.../src/site/markdown/MapredCommands.md | 4 +-
.../hadoop/mapred/TestFileInputFormat.java | 2 +-
.../jobhistory/TestHistoryViewerPrinter.java | 945 +++++++++++++
.../lib/input/TestFileInputFormat.java | 2 +-
.../lib/output/TestFileOutputCommitter.java | 4 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 6 +-
.../hadoop/mapreduce/v2/hs/JobHistory.java | 3 +-
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +-
.../mapreduce/v2/hs/webapp/HsJobsBlock.java | 2 +-
.../mapreduce/v2/hs/webapp/dao/JobInfo.java | 12 +
.../mapreduce/v2/hs/webapp/dao/TestJobInfo.java | 26 +-
.../org/apache/hadoop/mapred/ClientCache.java | 24 +-
.../hadoop/mapred/ClientServiceDelegate.java | 16 +
.../hadoop/mapred/ResourceMgrDelegate.java | 7 +
.../org/apache/hadoop/mapred/YARNRunner.java | 11 +
.../mapred/YarnClientProtocolProvider.java | 5 +-
.../hadoop/mapred/TestClientRedirect.java | 8 +
.../hadoop/mapreduce/TestMRJobClient.java | 160 ++-
.../TestYarnClientProtocolProvider.java | 6 +-
.../mapreduce/TimelineServicePerformance.java | 2 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 6 +-
.../org/apache/hadoop/examples/pi/Util.java | 5 +-
hadoop-project/pom.xml | 13 +-
hadoop-project/src/site/site.xml | 2 +
.../org/apache/hadoop/fs/s3a/Constants.java | 3 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 267 ++--
.../src/site/markdown/tools/hadoop-aws/index.md | 54 +
.../hadoop/fs/s3a/TestS3AConfiguration.java | 151 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 11 +-
.../fs/s3a/scale/TestS3ADeleteManyFiles.java | 2 +-
.../fs/azure/AzureNativeFileSystemStore.java | 30 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 8 +-
.../hadoop/fs/azure/NativeFileSystemStore.java | 2 +
.../fs/azure/NativeAzureFileSystemBaseTest.java | 67 +
.../apache/hadoop/tools/DistCpOptionSwitch.java | 2 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 10 +-
.../src/site/markdown/DistCp.md.vm | 8 +-
.../apache/hadoop/tools/TestOptionsParser.java | 18 +-
hadoop-tools/hadoop-kafka/pom.xml | 2 -
hadoop-yarn-project/CHANGES.txt | 113 +-
.../dev-support/findbugs-exclude.xml | 4 +
.../yarn/api/ApplicationClientProtocol.java | 49 +-
.../protocolrecords/ReservationListRequest.java | 228 ++++
.../ReservationListResponse.java | 79 ++
.../hadoop/yarn/api/records/ReservationACL.java | 56 +
.../api/records/ReservationAllocationState.java | 191 +++
.../api/records/ResourceAllocationRequest.java | 123 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../main/proto/applicationclient_protocol.proto | 1 +
.../src/main/proto/yarn_protos.proto | 17 +
.../src/main/proto/yarn_service_protos.proto | 12 +
.../applications/distributedshell/Client.java | 3 +-
.../hadoop/yarn/client/api/YarnClient.java | 49 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 1 +
.../yarn/client/api/impl/YarnClientImpl.java | 10 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 25 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 18 +-
.../yarn/client/util/YarnClientUtils.java | 113 ++
.../hadoop/yarn/client/util/package-info.java | 20 +
.../yarn/client/api/impl/TestYarnClient.java | 173 +++
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 32 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 9 +-
.../yarn/client/util/TestYarnClientUtils.java | 319 +++++
.../ApplicationClientProtocolPBClientImpl.java | 17 +
.../ApplicationClientProtocolPBServiceImpl.java | 24 +-
.../impl/pb/ReservationListRequestPBImpl.java | 178 +++
.../impl/pb/ReservationListResponsePBImpl.java | 157 +++
.../pb/ReservationAllocationStatePBImpl.java | 288 ++++
.../pb/ResourceAllocationRequestPBImpl.java | 188 +++
.../RequestHedgingRMFailoverProxyProvider.java | 4 +-
.../hadoop/yarn/webapp/hamlet/Hamlet.java | 11 +-
.../src/main/resources/yarn-default.xml | 56 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 22 +-
.../apache/hadoop/yarn/util/TestFSDownload.java | 15 +-
.../server/services/RegistryAdminService.java | 4 +-
.../server/nodemanager/DeletionService.java | 35 +-
.../containermanager/ContainerManagerImpl.java | 14 +
.../launcher/ContainersLauncher.java | 4 +-
.../localizer/ContainerLocalizer.java | 4 +-
.../localizer/ResourceLocalizationService.java | 8 +-
.../sharedcache/SharedCacheUploadService.java | 4 +-
.../logaggregation/LogAggregationService.java | 4 +-
.../loghandler/NonAggregatingLogHandler.java | 3 +-
.../recovery/NMLeveldbStateStoreService.java | 44 +
.../recovery/NMStateStoreService.java | 11 +
.../impl/container-executor.c | 231 ++--
.../test/test-container-executor.c | 10 +-
.../nodemanager/TestNodeStatusUpdater.java | 3 +-
.../amrmproxy/MockResourceManagerFacade.java | 14 +-
.../server/resourcemanager/AdminService.java | 6 +-
.../server/resourcemanager/ClientRMService.java | 158 ++-
.../resourcemanager/NodesListManager.java | 122 +-
.../server/resourcemanager/RMAuditLogger.java | 2 +
.../server/resourcemanager/ResourceManager.java | 5 +
.../blacklist/SimpleBlacklistManager.java | 9 +-
.../recovery/FileSystemRMStateStore.java | 2 +-
.../recovery/LeveldbRMStateStore.java | 2 +-
.../recovery/MemoryRMStateStore.java | 2 +-
.../recovery/NullRMStateStore.java | 2 +-
.../resourcemanager/recovery/RMStateStore.java | 2 +-
.../RMStateStoreStoreReservationEvent.java | 2 +-
.../recovery/ZKRMStateStore.java | 2 +-
.../reservation/AbstractReservationSystem.java | 16 +-
.../reservation/CapacityOverTimePolicy.java | 18 +-
.../reservation/InMemoryPlan.java | 77 +-
.../resourcemanager/reservation/PlanView.java | 31 +
.../reservation/ReservationInputValidator.java | 122 +-
.../ReservationSchedulerConfiguration.java | 15 +
.../reservation/ReservationSystem.java | 9 +
.../reservation/ReservationSystemUtil.java | 45 +-
.../planning/AlignedPlannerWithGreedy.java | 4 +-
.../planning/GreedyReservationAgent.java | 40 +-
.../reservation/planning/IterativePlanner.java | 239 ++--
.../planning/StageAllocatorGreedyRLE.java | 245 ++++
.../rmapp/attempt/RMAppAttempt.java | 6 +
.../rmapp/attempt/RMAppAttemptImpl.java | 40 +-
.../rmcontainer/AllocationExpirationInfo.java | 75 +
.../rmcontainer/ContainerAllocationExpirer.java | 11 +-
.../rmcontainer/RMContainer.java | 2 +
.../rmcontainer/RMContainerImpl.java | 132 +-
.../RMContainerNMDoneChangeResourceEvent.java | 37 +
.../resourcemanager/rmnode/RMNodeImpl.java | 31 +-
.../scheduler/AbstractYarnScheduler.java | 56 +-
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../scheduler/capacity/CapacityScheduler.java | 43 +-
.../CapacitySchedulerConfiguration.java | 38 +
.../scheduler/capacity/LeafQueue.java | 82 +-
.../event/ContainerExpiredSchedulerEvent.java | 12 +-
.../scheduler/fair/AllocationConfiguration.java | 20 +-
.../fair/AllocationFileLoaderService.java | 28 +-
.../scheduler/fair/FSAppAttempt.java | 7 +-
.../scheduler/fair/FSParentQueue.java | 6 +-
.../FifoOrderingPolicyForPendingApps.java | 73 +
.../scheduler/policy/RecoveryComparator.java | 33 +
.../scheduler/policy/SchedulableEntity.java | 10 +-
.../security/ReservationsACLsManager.java | 92 ++
.../webapp/RMAppAttemptBlock.java | 46 +-
.../resourcemanager/webapp/RMAppBlock.java | 32 +-
.../resourcemanager/webapp/RMAppsBlock.java | 5 +-
.../server/resourcemanager/webapp/RMWebApp.java | 5 +
.../resourcemanager/webapp/RMWebAppFilter.java | 90 ++
.../resourcemanager/webapp/RMWebServices.java | 97 +-
.../webapp/dao/AMBlackListingRequestInfo.java | 61 +
.../webapp/dao/AppAttemptInfo.java | 5 +
.../dao/ApplicationSubmissionContextInfo.java | 51 +
.../webapp/dao/LogAggregationContextInfo.java | 108 ++
.../webapp/dao/ReservationDefinitionInfo.java | 10 +
.../webapp/dao/ReservationIdInfo.java | 64 +
.../webapp/dao/ReservationInfo.java | 105 ++
.../webapp/dao/ReservationListInfo.java | 53 +
.../webapp/dao/ReservationRequestInfo.java | 9 +
.../webapp/dao/ReservationRequestsInfo.java | 11 +
.../webapp/dao/ResourceAllocationInfo.java | 72 +
.../yarn_server_resourcemanager_recovery.proto | 16 -
.../server/resourcemanager/ACLsTestBase.java | 123 ++
.../server/resourcemanager/Application.java | 10 +-
.../yarn/server/resourcemanager/MockNM.java | 20 +
.../yarn/server/resourcemanager/MockRM.java | 48 +-
.../resourcemanager/QueueACLsTestBase.java | 87 +-
.../ReservationACLsTestBase.java | 600 ++++++++
.../resourcemanager/TestClientRMService.java | 179 +++
.../resourcemanager/TestRMAdminService.java | 35 +-
.../resourcemanager/TestRMNodeTransitions.java | 18 +-
.../server/resourcemanager/TestRMRestart.java | 11 +-
.../TestReservationSystemWithRMHA.java | 2 +-
.../TestResourceTrackerService.java | 189 ++-
.../recovery/RMStateStoreTestBase.java | 6 +-
.../reservation/ReservationSystemTestUtil.java | 19 +-
.../reservation/TestInMemoryPlan.java | 314 +++--
.../TestReservationInputValidator.java | 103 +-
.../reservation/TestReservationSystemUtil.java | 134 ++
.../planning/TestGreedyReservationAgent.java | 120 +-
.../attempt/TestRMAppAttemptTransitions.java | 78 +-
.../rmcontainer/TestRMContainerImpl.java | 112 +-
.../scheduler/TestAbstractYarnScheduler.java | 5 +-
.../capacity/TestContainerResizing.java | 44 +-
.../capacity/TestIncreaseAllocationExpirer.java | 443 ++++++
.../scheduler/capacity/TestLeafQueue.java | 2 -
.../scheduler/capacity/TestUtils.java | 15 +-
.../scheduler/fair/TestFSLeafQueue.java | 4 +-
.../scheduler/policy/MockSchedulableEntity.java | 17 +
.../TestFifoOrderingPolicyForPendingApps.java | 89 ++
.../webapp/TestRMWebServicesApps.java | 2 +-
.../TestRMWebServicesAppsModification.java | 74 +-
.../webapp/TestRMWebServicesReservation.java | 542 +++++++-
.../src/test/resources/submit-reservation.json | 6 +-
.../sharedcachemanager/CleanerService.java | 4 +-
.../store/InMemorySCMStore.java | 4 +-
.../store/TestInMemorySCMStore.java | 7 +-
.../src/site/markdown/FairScheduler.md | 2 +-
.../src/site/markdown/NodeLabel.md | 86 +-
.../src/site/markdown/ResourceManagerRest.md | 54 +-
.../src/site/markdown/SecureContainer.md | 2 +-
.../src/site/markdown/TimelineServer.md | 13 +-
.../site/markdown/YarnApplicationSecurity.md | 560 ++++++++
492 files changed, 25814 insertions(+), 5889 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a140b213/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a140b213/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
[34/50] [abbrv] hadoop git commit: Move HDFS-8923 to Release 2.8.0 in
CHANGES.txt.
Posted by ar...@apache.org.
Move HDFS-8923 to Release 2.8.0 in CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/748b6c07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/748b6c07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/748b6c07
Branch: refs/heads/HDFS-1312
Commit: 748b6c07fe29bf52510d2e84d82fcbef7f1bb35c
Parents: 46f7790
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Feb 18 18:43:06 2016 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Feb 18 18:43:06 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/748b6c07/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 945c911..2129a69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1939,6 +1939,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9773. Remove dead code related to SimulatedFSDataset in tests.
(Brahma Reddy Battula via aajisaka)
+ HDFS-8923. Add -source flag to balancer usage message. (Chris Trezzo
+ via szetszwo)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
@@ -2817,9 +2820,6 @@ Release 2.7.3 - UNRELEASED
HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be
configurable (James Kinley and Nathan Roberts via kihwal)
- HDFS-8923. Add -source flag to balancer usage message. (Chris Trezzo
- via szetszwo)
-
OPTIMIZATIONS
BUG FIXES
[14/50] [abbrv] hadoop git commit: HDFS-9815. Move o.a.h.fs.Hdfs to
hadoop-hdfs-client. Contributed by Vinayakumar B.
Posted by ar...@apache.org.
HDFS-9815. Move o.a.h.fs.Hdfs to hadoop-hdfs-client. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77f7ca3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77f7ca3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77f7ca3e
Branch: refs/heads/HDFS-1312
Commit: 77f7ca3e94ecaa442c007b12d4ad773b37da097c
Parents: fd1befb
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Feb 17 10:56:11 2016 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Wed Feb 17 10:56:11 2016 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/Hdfs.java | 535 +++++++++++++++++++
.../java/org/apache/hadoop/fs/SWebHdfs.java | 64 +++
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 63 +++
.../main/java/org/apache/hadoop/fs/package.html | 26 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../main/java/org/apache/hadoop/fs/Hdfs.java | 535 -------------------
.../java/org/apache/hadoop/fs/SWebHdfs.java | 64 ---
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 63 ---
.../main/java/org/apache/hadoop/fs/package.html | 26 -
9 files changed, 691 insertions(+), 688 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
new file mode 100644
index 0000000..6dd8884
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -0,0 +1,535 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.DFSOutputStream;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
+import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.util.Progressable;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class Hdfs extends AbstractFileSystem {
+
+ DFSClient dfs;
+ private boolean verifyChecksum = true;
+
+ static {
+ HdfsConfiguration.init();
+ }
+
+ /**
+ * This constructor has the signature needed by
+ * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}
+ *
+ * @param theUri which must be that of Hdfs
+ * @param conf configuration
+ * @throws IOException
+ */
+ Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException {
+ super(theUri, HdfsConstants.HDFS_URI_SCHEME, true,
+ HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
+
+ if (!theUri.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) {
+ throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs");
+ }
+ String host = theUri.getHost();
+ if (host == null) {
+ throw new IOException("Incomplete HDFS URI, no host: " + theUri);
+ }
+
+ this.dfs = new DFSClient(theUri, conf, getStatistics());
+ }
+
+ @Override
+ public int getUriDefaultPort() {
+ return HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
+ }
+
+ @Override
+ public HdfsDataOutputStream createInternal(Path f,
+ EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
+ int bufferSize, short replication, long blockSize, Progressable progress,
+ ChecksumOpt checksumOpt, boolean createParent) throws IOException {
+
+ final DFSOutputStream dfsos = dfs.primitiveCreate(getUriPath(f),
+ absolutePermission, createFlag, createParent, replication, blockSize,
+ progress, bufferSize, checksumOpt);
+ return dfs.createWrappedOutputStream(dfsos, statistics,
+ dfsos.getInitialLen());
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive)
+ throws IOException, UnresolvedLinkException {
+ return dfs.delete(getUriPath(f), recursive);
+ }
+
+ @Override
+ public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
+ throws IOException, UnresolvedLinkException {
+ return dfs.getBlockLocations(getUriPath(p), start, len);
+ }
+
+ @Override
+ public FileChecksum getFileChecksum(Path f)
+ throws IOException, UnresolvedLinkException {
+ return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f)
+ throws IOException, UnresolvedLinkException {
+ HdfsFileStatus fi = dfs.getFileInfo(getUriPath(f));
+ if (fi != null) {
+ return fi.makeQualified(getUri(), f);
+ } else {
+ throw new FileNotFoundException("File does not exist: " + f.toString());
+ }
+ }
+
+ @Override
+ public FileStatus getFileLinkStatus(Path f)
+ throws IOException, UnresolvedLinkException {
+ HdfsFileStatus fi = dfs.getFileLinkInfo(getUriPath(f));
+ if (fi != null) {
+ return fi.makeQualified(getUri(), f);
+ } else {
+ throw new FileNotFoundException("File does not exist: " + f);
+ }
+ }
+
+ @Override
+ public FsStatus getFsStatus() throws IOException {
+ return dfs.getDiskStatus();
+ }
+
+ @Override
+ public FsServerDefaults getServerDefaults() throws IOException {
+ return dfs.getServerDefaults();
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(
+ final Path p)
+ throws FileNotFoundException, IOException {
+ return new DirListingIterator<LocatedFileStatus>(p, true) {
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ return ((HdfsLocatedFileStatus)getNext()).makeQualifiedLocated(
+ getUri(), p);
+ }
+ };
+ }
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ return new DirListingIterator<FileStatus>(f, false) {
+
+ @Override
+ public FileStatus next() throws IOException {
+ return getNext().makeQualified(getUri(), f);
+ }
+ };
+ }
+
+ /**
+ * This class defines an iterator that returns
+ * the file status of each file/subdirectory of a directory
+ *
+ * if needLocation, status contains block location if it is a file
+ * throws a RuntimeException with the error as its cause.
+ *
+ * @param <T> the type of the file status
+ */
+ abstract private class DirListingIterator<T extends FileStatus>
+ implements RemoteIterator<T> {
+ private DirectoryListing thisListing;
+ private int i;
+ final private String src;
+ final private boolean needLocation; // if status
+
+ private DirListingIterator(Path p, boolean needLocation)
+ throws IOException {
+ this.src = Hdfs.this.getUriPath(p);
+ this.needLocation = needLocation;
+
+ // fetch the first batch of entries in the directory
+ thisListing = dfs.listPaths(
+ src, HdfsFileStatus.EMPTY_NAME, needLocation);
+ if (thisListing == null) { // the directory does not exist
+ throw new FileNotFoundException("File " + src + " does not exist.");
+ }
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ if (thisListing == null) {
+ return false;
+ }
+ if (i>=thisListing.getPartialListing().length
+ && thisListing.hasMore()) {
+ // current listing is exhausted & fetch a new listing
+ thisListing = dfs.listPaths(src, thisListing.getLastName(),
+ needLocation);
+ if (thisListing == null) {
+ return false; // the directory is deleted
+ }
+ i = 0;
+ }
+ return (i<thisListing.getPartialListing().length);
+ }
+
+ /**
+ * Get the next item in the list
+ * @return the next item in the list
+ *
+ * @throws IOException if there is any error
+ * @throws NoSuchElementException if no more entry is available
+ */
+ public HdfsFileStatus getNext() throws IOException {
+ if (hasNext()) {
+ return thisListing.getPartialListing()[i++];
+ }
+ throw new NoSuchElementException("No more entry in " + src);
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f)
+ throws IOException, UnresolvedLinkException {
+ String src = getUriPath(f);
+
+ // fetch the first batch of entries in the directory
+ DirectoryListing thisListing = dfs.listPaths(
+ src, HdfsFileStatus.EMPTY_NAME);
+
+ if (thisListing == null) { // the directory does not exist
+ throw new FileNotFoundException("File " + f + " does not exist.");
+ }
+
+ HdfsFileStatus[] partialListing = thisListing.getPartialListing();
+ if (!thisListing.hasMore()) { // got all entries of the directory
+ FileStatus[] stats = new FileStatus[partialListing.length];
+ for (int i = 0; i < partialListing.length; i++) {
+ stats[i] = partialListing[i].makeQualified(getUri(), f);
+ }
+ return stats;
+ }
+
+ // The directory size is too big that it needs to fetch more
+ // estimate the total number of entries in the directory
+ int totalNumEntries =
+ partialListing.length + thisListing.getRemainingEntries();
+ ArrayList<FileStatus> listing =
+ new ArrayList<FileStatus>(totalNumEntries);
+ // add the first batch of entries to the array list
+ for (HdfsFileStatus fileStatus : partialListing) {
+ listing.add(fileStatus.makeQualified(getUri(), f));
+ }
+
+ // now fetch more entries
+ do {
+ thisListing = dfs.listPaths(src, thisListing.getLastName());
+
+ if (thisListing == null) {
+ // the directory is deleted
+ throw new FileNotFoundException("File " + f + " does not exist.");
+ }
+
+ partialListing = thisListing.getPartialListing();
+ for (HdfsFileStatus fileStatus : partialListing) {
+ listing.add(fileStatus.makeQualified(getUri(), f));
+ }
+ } while (thisListing.hasMore());
+
+ return listing.toArray(new FileStatus[listing.size()]);
+ }
+
+ @Override
+ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
+ throws IOException {
+ return new CorruptFileBlockIterator(dfs, path);
+ }
+
+ @Override
+ public void mkdir(Path dir, FsPermission permission, boolean createParent)
+ throws IOException, UnresolvedLinkException {
+ dfs.primitiveMkdir(getUriPath(dir), permission, createParent);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public HdfsDataInputStream open(Path f, int bufferSize)
+ throws IOException, UnresolvedLinkException {
+ final DFSInputStream dfsis = dfs.open(getUriPath(f),
+ bufferSize, verifyChecksum);
+ return dfs.createWrappedInputStream(dfsis);
+ }
+
+ @Override
+ public boolean truncate(Path f, long newLength)
+ throws IOException, UnresolvedLinkException {
+ return dfs.truncate(getUriPath(f), newLength);
+ }
+
+ @Override
+ public void renameInternal(Path src, Path dst)
+ throws IOException, UnresolvedLinkException {
+ dfs.rename(getUriPath(src), getUriPath(dst), Options.Rename.NONE);
+ }
+
+ @Override
+ public void renameInternal(Path src, Path dst, boolean overwrite)
+ throws IOException, UnresolvedLinkException {
+ dfs.rename(getUriPath(src), getUriPath(dst),
+ overwrite ? Options.Rename.OVERWRITE : Options.Rename.NONE);
+ }
+
+ @Override
+ public void setOwner(Path f, String username, String groupname)
+ throws IOException, UnresolvedLinkException {
+ dfs.setOwner(getUriPath(f), username, groupname);
+ }
+
+ @Override
+ public void setPermission(Path f, FsPermission permission)
+ throws IOException, UnresolvedLinkException {
+ dfs.setPermission(getUriPath(f), permission);
+ }
+
+ @Override
+ public boolean setReplication(Path f, short replication)
+ throws IOException, UnresolvedLinkException {
+ return dfs.setReplication(getUriPath(f), replication);
+ }
+
+ @Override
+ public void setTimes(Path f, long mtime, long atime)
+ throws IOException, UnresolvedLinkException {
+ dfs.setTimes(getUriPath(f), mtime, atime);
+ }
+
+ @Override
+ public void setVerifyChecksum(boolean verifyChecksum)
+ throws IOException {
+ this.verifyChecksum = verifyChecksum;
+ }
+
+ @Override
+ public boolean supportsSymlinks() {
+ return true;
+ }
+
+ @Override
+ public void createSymlink(Path target, Path link, boolean createParent)
+ throws IOException, UnresolvedLinkException {
+ dfs.createSymlink(target.toString(), getUriPath(link), createParent);
+ }
+
+ @Override
+ public Path getLinkTarget(Path p) throws IOException {
+ return new Path(dfs.getLinkTarget(getUriPath(p)));
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ return dfs.getCanonicalServiceName();
+ }
+
+ @Override //AbstractFileSystem
+ public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
+ Token<DelegationTokenIdentifier> result = dfs
+ .getDelegationToken(renewer == null ? null : new Text(renewer));
+ List<Token<?>> tokenList = new ArrayList<Token<?>>();
+ tokenList.add(result);
+ return tokenList;
+ }
+
+ @Override
+ public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
+ throws IOException {
+ dfs.modifyAclEntries(getUriPath(path), aclSpec);
+ }
+
+ @Override
+ public void removeAclEntries(Path path, List<AclEntry> aclSpec)
+ throws IOException {
+ dfs.removeAclEntries(getUriPath(path), aclSpec);
+ }
+
+ @Override
+ public void removeDefaultAcl(Path path) throws IOException {
+ dfs.removeDefaultAcl(getUriPath(path));
+ }
+
+ @Override
+ public void removeAcl(Path path) throws IOException {
+ dfs.removeAcl(getUriPath(path));
+ }
+
+ @Override
+ public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
+ dfs.setAcl(getUriPath(path), aclSpec);
+ }
+
+ @Override
+ public AclStatus getAclStatus(Path path) throws IOException {
+ return dfs.getAclStatus(getUriPath(path));
+ }
+
+ @Override
+ public void setXAttr(Path path, String name, byte[] value,
+ EnumSet<XAttrSetFlag> flag) throws IOException {
+ dfs.setXAttr(getUriPath(path), name, value, flag);
+ }
+
+ @Override
+ public byte[] getXAttr(Path path, String name) throws IOException {
+ return dfs.getXAttr(getUriPath(path), name);
+ }
+
+ @Override
+ public Map<String, byte[]> getXAttrs(Path path) throws IOException {
+ return dfs.getXAttrs(getUriPath(path));
+ }
+
+ @Override
+ public Map<String, byte[]> getXAttrs(Path path, List<String> names)
+ throws IOException {
+ return dfs.getXAttrs(getUriPath(path), names);
+ }
+
+ @Override
+ public List<String> listXAttrs(Path path) throws IOException {
+ return dfs.listXAttrs(getUriPath(path));
+ }
+
+ @Override
+ public void removeXAttr(Path path, String name) throws IOException {
+ dfs.removeXAttr(getUriPath(path), name);
+ }
+
+ @Override
+ public void access(Path path, final FsAction mode) throws IOException {
+ dfs.checkAccess(getUriPath(path), mode);
+ }
+
+ @Override
+ public void setStoragePolicy(Path path, String policyName) throws IOException {
+ dfs.setStoragePolicy(getUriPath(path), policyName);
+ }
+
+ @Override
+ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
+ return dfs.getStoragePolicy(getUriPath(src));
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return Arrays.asList(dfs.getStoragePolicies());
+ }
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws InvalidToken
+ * @throws IOException
+ * @deprecated Use Token.renew instead.
+ */
+ @SuppressWarnings("unchecked")
+ public long renewDelegationToken(
+ Token<? extends AbstractDelegationTokenIdentifier> token)
+ throws InvalidToken, IOException {
+ return dfs.renewDelegationToken((Token<DelegationTokenIdentifier>) token);
+ }
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws InvalidToken
+ * @throws IOException
+ * @deprecated Use Token.cancel instead.
+ */
+ @SuppressWarnings("unchecked")
+ public void cancelDelegationToken(
+ Token<? extends AbstractDelegationTokenIdentifier> token)
+ throws InvalidToken, IOException {
+ dfs.cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
+ }
+
+ @Override
+ public Path createSnapshot(final Path path, final String snapshotName)
+ throws IOException {
+ return new Path(dfs.createSnapshot(getUriPath(path), snapshotName));
+ }
+
+ @Override
+ public void renameSnapshot(final Path path, final String snapshotOldName,
+ final String snapshotNewName) throws IOException {
+ dfs.renameSnapshot(getUriPath(path), snapshotOldName, snapshotNewName);
+ }
+
+ @Override
+ public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
+ throws IOException {
+ dfs.deleteSnapshot(getUriPath(snapshotDir), snapshotName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/SWebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
new file mode 100644
index 0000000..04b5880
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * AbstractFileSystem implementation for HDFS over the web (secure).
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class SWebHdfs extends DelegateToFileSystem {
+
+ public static final String SCHEME = "swebhdfs";
+
+ /**
+ * This constructor has the signature needed by
+ * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}
+ *
+ * @param theUri which must be that of swebhdfs
+ * @param conf configuration
+ * @throws IOException
+ */
+ SWebHdfs(URI theUri, Configuration conf)
+ throws IOException, URISyntaxException {
+ super(theUri, createSWebHdfsFileSystem(conf), conf, SCHEME, false);
+ }
+
+ /**
+ * Returns a new {@link SWebHdfsFileSystem}, with the given configuration.
+ *
+ * @param conf configuration
+ * @return new SWebHdfsFileSystem
+ */
+ private static SWebHdfsFileSystem createSWebHdfsFileSystem(
+ Configuration conf) {
+ SWebHdfsFileSystem fs = new SWebHdfsFileSystem();
+ fs.setConf(conf);
+ return fs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/WebHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/WebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/WebHdfs.java
new file mode 100644
index 0000000..d54a6b3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/WebHdfs.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * AbstractFileSystem implementation for HDFS over the web.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class WebHdfs extends DelegateToFileSystem {
+
+ public static final String SCHEME = "webhdfs";
+
+ /**
+ * This constructor has the signature needed by
+ * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}
+ *
+ * @param theUri which must be that of webhdfs
+ * @param conf configuration
+ * @throws IOException
+ */
+ WebHdfs(URI theUri, Configuration conf)
+ throws IOException, URISyntaxException {
+ super(theUri, createWebHdfsFileSystem(conf), conf, SCHEME, false);
+ }
+
+ /**
+ * Returns a new {@link WebHdfsFileSystem}, with the given configuration.
+ *
+ * @param conf configuration
+ * @return new WebHdfsFileSystem
+ */
+ private static WebHdfsFileSystem createWebHdfsFileSystem(Configuration conf) {
+ WebHdfsFileSystem fs = new WebHdfsFileSystem();
+ fs.setConf(conf);
+ return fs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/package.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/package.html b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/package.html
new file mode 100644
index 0000000..53b2a5a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/package.html
@@ -0,0 +1,26 @@
+<html>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<body>
+
+<p>Implementations of {@link org.apache.hadoop.fs.AbstractFileSystem} for hdfs
+ over rpc and hdfs over web.</p>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 79369da..5ca7f8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2774,6 +2774,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9456. BlockPlacementPolicyWithNodeGroup should override
verifyBlockPlacement(). (Xiaobing Zhou via junping_du)
+ HDFS-9815. Move o.a.h.fs.Hdfs to hadoop-hdfs-client.
+ (Vinayakumar B via wheat9)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
deleted file mode 100644
index 6dd8884..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ /dev/null
@@ -1,535 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.AclEntry;
-import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.Options.ChecksumOpt;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSInputStream;
-import org.apache.hadoop.hdfs.DFSOutputStream;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
-import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
-import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.util.Progressable;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class Hdfs extends AbstractFileSystem {
-
- DFSClient dfs;
- private boolean verifyChecksum = true;
-
- static {
- HdfsConfiguration.init();
- }
-
- /**
- * This constructor has the signature needed by
- * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}
- *
- * @param theUri which must be that of Hdfs
- * @param conf configuration
- * @throws IOException
- */
- Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException {
- super(theUri, HdfsConstants.HDFS_URI_SCHEME, true,
- HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
-
- if (!theUri.getScheme().equalsIgnoreCase(HdfsConstants.HDFS_URI_SCHEME)) {
- throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs");
- }
- String host = theUri.getHost();
- if (host == null) {
- throw new IOException("Incomplete HDFS URI, no host: " + theUri);
- }
-
- this.dfs = new DFSClient(theUri, conf, getStatistics());
- }
-
- @Override
- public int getUriDefaultPort() {
- return HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT;
- }
-
- @Override
- public HdfsDataOutputStream createInternal(Path f,
- EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
- int bufferSize, short replication, long blockSize, Progressable progress,
- ChecksumOpt checksumOpt, boolean createParent) throws IOException {
-
- final DFSOutputStream dfsos = dfs.primitiveCreate(getUriPath(f),
- absolutePermission, createFlag, createParent, replication, blockSize,
- progress, bufferSize, checksumOpt);
- return dfs.createWrappedOutputStream(dfsos, statistics,
- dfsos.getInitialLen());
- }
-
- @Override
- public boolean delete(Path f, boolean recursive)
- throws IOException, UnresolvedLinkException {
- return dfs.delete(getUriPath(f), recursive);
- }
-
- @Override
- public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
- throws IOException, UnresolvedLinkException {
- return dfs.getBlockLocations(getUriPath(p), start, len);
- }
-
- @Override
- public FileChecksum getFileChecksum(Path f)
- throws IOException, UnresolvedLinkException {
- return dfs.getFileChecksum(getUriPath(f), Long.MAX_VALUE);
- }
-
- @Override
- public FileStatus getFileStatus(Path f)
- throws IOException, UnresolvedLinkException {
- HdfsFileStatus fi = dfs.getFileInfo(getUriPath(f));
- if (fi != null) {
- return fi.makeQualified(getUri(), f);
- } else {
- throw new FileNotFoundException("File does not exist: " + f.toString());
- }
- }
-
- @Override
- public FileStatus getFileLinkStatus(Path f)
- throws IOException, UnresolvedLinkException {
- HdfsFileStatus fi = dfs.getFileLinkInfo(getUriPath(f));
- if (fi != null) {
- return fi.makeQualified(getUri(), f);
- } else {
- throw new FileNotFoundException("File does not exist: " + f);
- }
- }
-
- @Override
- public FsStatus getFsStatus() throws IOException {
- return dfs.getDiskStatus();
- }
-
- @Override
- public FsServerDefaults getServerDefaults() throws IOException {
- return dfs.getServerDefaults();
- }
-
- @Override
- public RemoteIterator<LocatedFileStatus> listLocatedStatus(
- final Path p)
- throws FileNotFoundException, IOException {
- return new DirListingIterator<LocatedFileStatus>(p, true) {
-
- @Override
- public LocatedFileStatus next() throws IOException {
- return ((HdfsLocatedFileStatus)getNext()).makeQualifiedLocated(
- getUri(), p);
- }
- };
- }
-
- @Override
- public RemoteIterator<FileStatus> listStatusIterator(final Path f)
- throws AccessControlException, FileNotFoundException,
- UnresolvedLinkException, IOException {
- return new DirListingIterator<FileStatus>(f, false) {
-
- @Override
- public FileStatus next() throws IOException {
- return getNext().makeQualified(getUri(), f);
- }
- };
- }
-
- /**
- * This class defines an iterator that returns
- * the file status of each file/subdirectory of a directory
- *
- * if needLocation, status contains block location if it is a file
- * throws a RuntimeException with the error as its cause.
- *
- * @param <T> the type of the file status
- */
- abstract private class DirListingIterator<T extends FileStatus>
- implements RemoteIterator<T> {
- private DirectoryListing thisListing;
- private int i;
- final private String src;
- final private boolean needLocation; // if status
-
- private DirListingIterator(Path p, boolean needLocation)
- throws IOException {
- this.src = Hdfs.this.getUriPath(p);
- this.needLocation = needLocation;
-
- // fetch the first batch of entries in the directory
- thisListing = dfs.listPaths(
- src, HdfsFileStatus.EMPTY_NAME, needLocation);
- if (thisListing == null) { // the directory does not exist
- throw new FileNotFoundException("File " + src + " does not exist.");
- }
- }
-
- @Override
- public boolean hasNext() throws IOException {
- if (thisListing == null) {
- return false;
- }
- if (i>=thisListing.getPartialListing().length
- && thisListing.hasMore()) {
- // current listing is exhausted & fetch a new listing
- thisListing = dfs.listPaths(src, thisListing.getLastName(),
- needLocation);
- if (thisListing == null) {
- return false; // the directory is deleted
- }
- i = 0;
- }
- return (i<thisListing.getPartialListing().length);
- }
-
- /**
- * Get the next item in the list
- * @return the next item in the list
- *
- * @throws IOException if there is any error
- * @throws NoSuchElementException if no more entry is available
- */
- public HdfsFileStatus getNext() throws IOException {
- if (hasNext()) {
- return thisListing.getPartialListing()[i++];
- }
- throw new NoSuchElementException("No more entry in " + src);
- }
- }
-
- @Override
- public FileStatus[] listStatus(Path f)
- throws IOException, UnresolvedLinkException {
- String src = getUriPath(f);
-
- // fetch the first batch of entries in the directory
- DirectoryListing thisListing = dfs.listPaths(
- src, HdfsFileStatus.EMPTY_NAME);
-
- if (thisListing == null) { // the directory does not exist
- throw new FileNotFoundException("File " + f + " does not exist.");
- }
-
- HdfsFileStatus[] partialListing = thisListing.getPartialListing();
- if (!thisListing.hasMore()) { // got all entries of the directory
- FileStatus[] stats = new FileStatus[partialListing.length];
- for (int i = 0; i < partialListing.length; i++) {
- stats[i] = partialListing[i].makeQualified(getUri(), f);
- }
- return stats;
- }
-
- // The directory size is too big that it needs to fetch more
- // estimate the total number of entries in the directory
- int totalNumEntries =
- partialListing.length + thisListing.getRemainingEntries();
- ArrayList<FileStatus> listing =
- new ArrayList<FileStatus>(totalNumEntries);
- // add the first batch of entries to the array list
- for (HdfsFileStatus fileStatus : partialListing) {
- listing.add(fileStatus.makeQualified(getUri(), f));
- }
-
- // now fetch more entries
- do {
- thisListing = dfs.listPaths(src, thisListing.getLastName());
-
- if (thisListing == null) {
- // the directory is deleted
- throw new FileNotFoundException("File " + f + " does not exist.");
- }
-
- partialListing = thisListing.getPartialListing();
- for (HdfsFileStatus fileStatus : partialListing) {
- listing.add(fileStatus.makeQualified(getUri(), f));
- }
- } while (thisListing.hasMore());
-
- return listing.toArray(new FileStatus[listing.size()]);
- }
-
- @Override
- public RemoteIterator<Path> listCorruptFileBlocks(Path path)
- throws IOException {
- return new CorruptFileBlockIterator(dfs, path);
- }
-
- @Override
- public void mkdir(Path dir, FsPermission permission, boolean createParent)
- throws IOException, UnresolvedLinkException {
- dfs.primitiveMkdir(getUriPath(dir), permission, createParent);
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public HdfsDataInputStream open(Path f, int bufferSize)
- throws IOException, UnresolvedLinkException {
- final DFSInputStream dfsis = dfs.open(getUriPath(f),
- bufferSize, verifyChecksum);
- return dfs.createWrappedInputStream(dfsis);
- }
-
- @Override
- public boolean truncate(Path f, long newLength)
- throws IOException, UnresolvedLinkException {
- return dfs.truncate(getUriPath(f), newLength);
- }
-
- @Override
- public void renameInternal(Path src, Path dst)
- throws IOException, UnresolvedLinkException {
- dfs.rename(getUriPath(src), getUriPath(dst), Options.Rename.NONE);
- }
-
- @Override
- public void renameInternal(Path src, Path dst, boolean overwrite)
- throws IOException, UnresolvedLinkException {
- dfs.rename(getUriPath(src), getUriPath(dst),
- overwrite ? Options.Rename.OVERWRITE : Options.Rename.NONE);
- }
-
- @Override
- public void setOwner(Path f, String username, String groupname)
- throws IOException, UnresolvedLinkException {
- dfs.setOwner(getUriPath(f), username, groupname);
- }
-
- @Override
- public void setPermission(Path f, FsPermission permission)
- throws IOException, UnresolvedLinkException {
- dfs.setPermission(getUriPath(f), permission);
- }
-
- @Override
- public boolean setReplication(Path f, short replication)
- throws IOException, UnresolvedLinkException {
- return dfs.setReplication(getUriPath(f), replication);
- }
-
- @Override
- public void setTimes(Path f, long mtime, long atime)
- throws IOException, UnresolvedLinkException {
- dfs.setTimes(getUriPath(f), mtime, atime);
- }
-
- @Override
- public void setVerifyChecksum(boolean verifyChecksum)
- throws IOException {
- this.verifyChecksum = verifyChecksum;
- }
-
- @Override
- public boolean supportsSymlinks() {
- return true;
- }
-
- @Override
- public void createSymlink(Path target, Path link, boolean createParent)
- throws IOException, UnresolvedLinkException {
- dfs.createSymlink(target.toString(), getUriPath(link), createParent);
- }
-
- @Override
- public Path getLinkTarget(Path p) throws IOException {
- return new Path(dfs.getLinkTarget(getUriPath(p)));
- }
-
- @Override
- public String getCanonicalServiceName() {
- return dfs.getCanonicalServiceName();
- }
-
- @Override //AbstractFileSystem
- public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
- Token<DelegationTokenIdentifier> result = dfs
- .getDelegationToken(renewer == null ? null : new Text(renewer));
- List<Token<?>> tokenList = new ArrayList<Token<?>>();
- tokenList.add(result);
- return tokenList;
- }
-
- @Override
- public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
- throws IOException {
- dfs.modifyAclEntries(getUriPath(path), aclSpec);
- }
-
- @Override
- public void removeAclEntries(Path path, List<AclEntry> aclSpec)
- throws IOException {
- dfs.removeAclEntries(getUriPath(path), aclSpec);
- }
-
- @Override
- public void removeDefaultAcl(Path path) throws IOException {
- dfs.removeDefaultAcl(getUriPath(path));
- }
-
- @Override
- public void removeAcl(Path path) throws IOException {
- dfs.removeAcl(getUriPath(path));
- }
-
- @Override
- public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
- dfs.setAcl(getUriPath(path), aclSpec);
- }
-
- @Override
- public AclStatus getAclStatus(Path path) throws IOException {
- return dfs.getAclStatus(getUriPath(path));
- }
-
- @Override
- public void setXAttr(Path path, String name, byte[] value,
- EnumSet<XAttrSetFlag> flag) throws IOException {
- dfs.setXAttr(getUriPath(path), name, value, flag);
- }
-
- @Override
- public byte[] getXAttr(Path path, String name) throws IOException {
- return dfs.getXAttr(getUriPath(path), name);
- }
-
- @Override
- public Map<String, byte[]> getXAttrs(Path path) throws IOException {
- return dfs.getXAttrs(getUriPath(path));
- }
-
- @Override
- public Map<String, byte[]> getXAttrs(Path path, List<String> names)
- throws IOException {
- return dfs.getXAttrs(getUriPath(path), names);
- }
-
- @Override
- public List<String> listXAttrs(Path path) throws IOException {
- return dfs.listXAttrs(getUriPath(path));
- }
-
- @Override
- public void removeXAttr(Path path, String name) throws IOException {
- dfs.removeXAttr(getUriPath(path), name);
- }
-
- @Override
- public void access(Path path, final FsAction mode) throws IOException {
- dfs.checkAccess(getUriPath(path), mode);
- }
-
- @Override
- public void setStoragePolicy(Path path, String policyName) throws IOException {
- dfs.setStoragePolicy(getUriPath(path), policyName);
- }
-
- @Override
- public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
- return dfs.getStoragePolicy(getUriPath(src));
- }
-
- @Override
- public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
- throws IOException {
- return Arrays.asList(dfs.getStoragePolicies());
- }
-
- /**
- * Renew an existing delegation token.
- *
- * @param token delegation token obtained earlier
- * @return the new expiration time
- * @throws InvalidToken
- * @throws IOException
- * @deprecated Use Token.renew instead.
- */
- @SuppressWarnings("unchecked")
- public long renewDelegationToken(
- Token<? extends AbstractDelegationTokenIdentifier> token)
- throws InvalidToken, IOException {
- return dfs.renewDelegationToken((Token<DelegationTokenIdentifier>) token);
- }
-
- /**
- * Cancel an existing delegation token.
- *
- * @param token delegation token
- * @throws InvalidToken
- * @throws IOException
- * @deprecated Use Token.cancel instead.
- */
- @SuppressWarnings("unchecked")
- public void cancelDelegationToken(
- Token<? extends AbstractDelegationTokenIdentifier> token)
- throws InvalidToken, IOException {
- dfs.cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
- }
-
- @Override
- public Path createSnapshot(final Path path, final String snapshotName)
- throws IOException {
- return new Path(dfs.createSnapshot(getUriPath(path), snapshotName));
- }
-
- @Override
- public void renameSnapshot(final Path path, final String snapshotOldName,
- final String snapshotNewName) throws IOException {
- dfs.renameSnapshot(getUriPath(path), snapshotOldName, snapshotNewName);
- }
-
- @Override
- public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
- throws IOException {
- dfs.deleteSnapshot(getUriPath(snapshotDir), snapshotName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
deleted file mode 100644
index 04b5880..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * AbstractFileSystem implementation for HDFS over the web (secure).
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class SWebHdfs extends DelegateToFileSystem {
-
- public static final String SCHEME = "swebhdfs";
-
- /**
- * This constructor has the signature needed by
- * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}
- *
- * @param theUri which must be that of swebhdfs
- * @param conf configuration
- * @throws IOException
- */
- SWebHdfs(URI theUri, Configuration conf)
- throws IOException, URISyntaxException {
- super(theUri, createSWebHdfsFileSystem(conf), conf, SCHEME, false);
- }
-
- /**
- * Returns a new {@link SWebHdfsFileSystem}, with the given configuration.
- *
- * @param conf configuration
- * @return new SWebHdfsFileSystem
- */
- private static SWebHdfsFileSystem createSWebHdfsFileSystem(
- Configuration conf) {
- SWebHdfsFileSystem fs = new SWebHdfsFileSystem();
- fs.setConf(conf);
- return fs;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/WebHdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/WebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/WebHdfs.java
deleted file mode 100644
index d54a6b3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/WebHdfs.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * AbstractFileSystem implementation for HDFS over the web.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class WebHdfs extends DelegateToFileSystem {
-
- public static final String SCHEME = "webhdfs";
-
- /**
- * This constructor has the signature needed by
- * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}
- *
- * @param theUri which must be that of webhdfs
- * @param conf configuration
- * @throws IOException
- */
- WebHdfs(URI theUri, Configuration conf)
- throws IOException, URISyntaxException {
- super(theUri, createWebHdfsFileSystem(conf), conf, SCHEME, false);
- }
-
- /**
- * Returns a new {@link WebHdfsFileSystem}, with the given configuration.
- *
- * @param conf configuration
- * @return new WebHdfsFileSystem
- */
- private static WebHdfsFileSystem createWebHdfsFileSystem(Configuration conf) {
- WebHdfsFileSystem fs = new WebHdfsFileSystem();
- fs.setConf(conf);
- return fs;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77f7ca3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/package.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/package.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/package.html
deleted file mode 100644
index 53b2a5a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/package.html
+++ /dev/null
@@ -1,26 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-
-<p>Implementations of {@link org.apache.hadoop.fs.AbstractFileSystem} for hdfs
- over rpc and hdfs over web.</p>
-
-</body>
-</html>
[32/50] [abbrv] hadoop git commit: MAPREDUCE-6627. Add
machine-readable output to mapred job -history command (rkanter)
Posted by ar...@apache.org.
MAPREDUCE-6627. Add machine-readable output to mapred job -history command (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8eee59ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8eee59ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8eee59ce
Branch: refs/heads/HDFS-1312
Commit: 8eee59ce6b3044cb73bb41fed6b7ece959e7c2f8
Parents: 8ab7658
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Feb 18 17:58:26 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Feb 18 17:58:26 2016 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../hadoop-mapreduce-client-core/pom.xml | 5 +
.../mapreduce/jobhistory/HistoryViewer.java | 480 +---------
.../jobhistory/HistoryViewerPrinter.java | 39 +
.../HumanReadableHistoryViewerPrinter.java | 471 +++++++++
.../jobhistory/JSONHistoryViewerPrinter.java | 256 +++++
.../org/apache/hadoop/mapreduce/tools/CLI.java | 79 +-
.../src/site/markdown/MapredCommands.md | 4 +-
.../jobhistory/TestHistoryViewerPrinter.java | 945 +++++++++++++++++++
.../hadoop/mapreduce/TestMRJobClient.java | 160 +++-
hadoop-project/pom.xml | 6 +
11 files changed, 1986 insertions(+), 462 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index da28bc6..a6031c0 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -313,6 +313,9 @@ Release 2.9.0 - UNRELEASED
MAPREDUCE-6634. Log uncaught exceptions/errors in various thread pools in
mapreduce. (Sidharta Seethana via vvasudev)
+ MAPREDUCE-6627. Add machine-readable output to mapred job -history
+ command (rkanter)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 61f3c53..f0952bc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -42,6 +42,11 @@
<artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.skyscreamer</groupId>
+ <artifactId>jsonassert</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
index f343d7c..e5db2e5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
@@ -19,13 +19,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.text.DecimalFormat;
-import java.text.Format;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Comparator;
+import java.io.PrintStream;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@@ -37,46 +32,52 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.TaskStatus;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
import org.apache.hadoop.mapreduce.util.HostUtil;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
/**
- * HistoryViewer is used to parse and view the JobHistory files
- *
+ * HistoryViewer is used to parse and view the JobHistory files. They can be
+ * printed in human-readable format or machine-readable JSON format using the
+ * {@link HistoryViewerPrinter}.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class HistoryViewer {
private static final Log LOG = LogFactory.getLog(HistoryViewer.class);
- private static final SimpleDateFormat dateFormat =
- new SimpleDateFormat("d-MMM-yyyy HH:mm:ss");
private FileSystem fs;
private JobInfo job;
- private String jobId;
- private boolean printAll;
+ private HistoryViewerPrinter jhvp;
+ public static final String HUMAN_FORMAT = "human";
+ public static final String JSON_FORMAT = "json";
-/**
- * Constructs the HistoryViewer object
- * @param historyFile The fully qualified Path of the History File
- * @param conf The Configuration file
- * @param printAll Toggle to print all status to only killed/failed status
- * @throws IOException
- */
- public HistoryViewer(String historyFile,
- Configuration conf,
+ /**
+ * Constructs the HistoryViewer object.
+ * @param historyFile the fully qualified Path of the History File
+ * @param conf the Configuration file
+ * @param printAll toggle to print all status to only killed/failed status
+ * @throws IOException when there is a problem parsing the history file
+ */
+ public HistoryViewer(String historyFile, Configuration conf,
boolean printAll) throws IOException {
- this.printAll = printAll;
+ this(historyFile, conf, printAll, HUMAN_FORMAT);
+ }
+
+ /**
+ * Constructs the HistoryViewer object.
+ * @param historyFile the fully qualified Path of the History File
+ * @param conf the Configuration file
+ * @param printAll toggle to print all status to only killed/failed status
+ * @param format the output format to use
+ * @throws IOException when there is a problem parsing the history file
+ */
+ public HistoryViewer(String historyFile, Configuration conf, boolean printAll,
+ String format) throws IOException {
String errorMsg = "Unable to initialize History Viewer";
try {
Path jobFile = new Path(historyFile);
@@ -101,366 +102,41 @@ public class HistoryViewer {
}
JobHistoryParser parser = new JobHistoryParser(fs, jobFile);
job = parser.parse();
- jobId = job.getJobId().toString();
- } catch(Exception e) {
+ String scheme = WebAppUtils.getHttpSchemePrefix(fs.getConf());
+ if (HUMAN_FORMAT.equalsIgnoreCase(format)) {
+ jhvp = new HumanReadableHistoryViewerPrinter(job, printAll, scheme);
+ } else if (JSON_FORMAT.equalsIgnoreCase(format)) {
+ jhvp = new JSONHistoryViewerPrinter(job, printAll, scheme);
+ } else {
+ System.err.println("Invalid format specified: " + format);
+ throw new IllegalArgumentException(errorMsg);
+ }
+ } catch(IOException e) {
throw new IOException(errorMsg, e);
}
}
/**
- * Print the job/task/attempt summary information
- * @throws IOException
+ * Print the job/task/attempt summary information to stdout.
+ * @throws IOException when there is a problem printing the history
*/
- public void print() throws IOException{
- printJobDetails();
- printTaskSummary();
- printJobAnalysis();
- printTasks(TaskType.JOB_SETUP, TaskStatus.State.FAILED.toString());
- printTasks(TaskType.JOB_SETUP, TaskStatus.State.KILLED.toString());
- printTasks(TaskType.MAP, TaskStatus.State.FAILED.toString());
- printTasks(TaskType.MAP, TaskStatus.State.KILLED.toString());
- printTasks(TaskType.REDUCE, TaskStatus.State.FAILED.toString());
- printTasks(TaskType.REDUCE, TaskStatus.State.KILLED.toString());
- printTasks(TaskType.JOB_CLEANUP, TaskStatus.State.FAILED.toString());
- printTasks(TaskType.JOB_CLEANUP,
- JobStatus.getJobRunState(JobStatus.KILLED));
- if (printAll) {
- printTasks(TaskType.JOB_SETUP, TaskStatus.State.SUCCEEDED.toString());
- printTasks(TaskType.MAP, TaskStatus.State.SUCCEEDED.toString());
- printTasks(TaskType.REDUCE, TaskStatus.State.SUCCEEDED.toString());
- printTasks(TaskType.JOB_CLEANUP, TaskStatus.State.SUCCEEDED.toString());
- printAllTaskAttempts(TaskType.JOB_SETUP);
- printAllTaskAttempts(TaskType.MAP);
- printAllTaskAttempts(TaskType.REDUCE);
- printAllTaskAttempts(TaskType.JOB_CLEANUP);
- }
-
- FilteredJob filter = new FilteredJob(job,
- TaskStatus.State.FAILED.toString());
- printFailedAttempts(filter);
-
- filter = new FilteredJob(job,
- TaskStatus.State.KILLED.toString());
- printFailedAttempts(filter);
- }
-
- private void printJobDetails() {
- StringBuffer jobDetails = new StringBuffer();
- jobDetails.append("\nHadoop job: " ).append(job.getJobId());
- jobDetails.append("\n=====================================");
- jobDetails.append("\nUser: ").append(job.getUsername());
- jobDetails.append("\nJobName: ").append(job.getJobname());
- jobDetails.append("\nJobConf: ").append(job.getJobConfPath());
- jobDetails.append("\nSubmitted At: ").append(StringUtils.
- getFormattedTimeWithDiff(dateFormat,
- job.getSubmitTime(), 0));
- jobDetails.append("\nLaunched At: ").append(StringUtils.
- getFormattedTimeWithDiff(dateFormat,
- job.getLaunchTime(),
- job.getSubmitTime()));
- jobDetails.append("\nFinished At: ").append(StringUtils.
- getFormattedTimeWithDiff(dateFormat,
- job.getFinishTime(),
- job.getLaunchTime()));
- jobDetails.append("\nStatus: ").append(((job.getJobStatus() == null) ?
- "Incomplete" :job.getJobStatus()));
- printCounters(jobDetails, job.getTotalCounters(), job.getMapCounters(),
- job.getReduceCounters());
- jobDetails.append("\n");
- jobDetails.append("\n=====================================");
- System.out.println(jobDetails.toString());
- }
-
- private void printCounters(StringBuffer buff, Counters totalCounters,
- Counters mapCounters, Counters reduceCounters) {
- // Killed jobs might not have counters
- if (totalCounters == null) {
- return;
- }
- buff.append("\nCounters: \n\n");
- buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|",
- "Group Name",
- "Counter name",
- "Map Value",
- "Reduce Value",
- "Total Value"));
- buff.append("\n------------------------------------------"+
- "---------------------------------------------");
- for (String groupName : totalCounters.getGroupNames()) {
- CounterGroup totalGroup = totalCounters.getGroup(groupName);
- CounterGroup mapGroup = mapCounters.getGroup(groupName);
- CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
-
- Format decimal = new DecimalFormat();
- Iterator<org.apache.hadoop.mapreduce.Counter> ctrItr =
- totalGroup.iterator();
- while(ctrItr.hasNext()) {
- org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
- String name = counter.getName();
- String mapValue =
- decimal.format(mapGroup.findCounter(name).getValue());
- String reduceValue =
- decimal.format(reduceGroup.findCounter(name).getValue());
- String totalValue =
- decimal.format(counter.getValue());
-
- buff.append(
- String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s",
- totalGroup.getDisplayName(),
- counter.getDisplayName(),
- mapValue, reduceValue, totalValue));
- }
- }
- }
-
- private void printAllTaskAttempts(TaskType taskType) {
- Map<TaskID, TaskInfo> tasks = job.getAllTasks();
- StringBuffer taskList = new StringBuffer();
- taskList.append("\n").append(taskType);
- taskList.append(" task list for ").append(job.getJobId());
- taskList.append("\nTaskId\t\tStartTime");
- if (TaskType.REDUCE.equals(taskType)) {
- taskList.append("\tShuffleFinished\tSortFinished");
- }
- taskList.append("\tFinishTime\tHostName\tError\tTaskLogs");
- taskList.append("\n====================================================");
- System.out.println(taskList.toString());
- for (JobHistoryParser.TaskInfo task : tasks.values()) {
- for (JobHistoryParser.TaskAttemptInfo attempt :
- task.getAllTaskAttempts().values()) {
- if (taskType.equals(task.getTaskType())){
- taskList.setLength(0);
- taskList.append(attempt.getAttemptId()).append("\t");
- taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
- attempt.getStartTime(), 0)).append("\t");
- if (TaskType.REDUCE.equals(taskType)) {
- taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
- attempt.getShuffleFinishTime(),
- attempt.getStartTime()));
- taskList.append("\t");
- taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
- attempt.getSortFinishTime(),
- attempt.getShuffleFinishTime()));
- }
- taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
- attempt.getFinishTime(),
- attempt.getStartTime()));
- taskList.append("\t");
- taskList.append(attempt.getHostname()).append("\t");
- taskList.append(attempt.getError());
- String taskLogsUrl = getTaskLogsUrl(
- WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt);
- taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a");
- System.out.println(taskList.toString());
- }
- }
- }
- }
-
- private void printTaskSummary() {
- SummarizedJob ts = new SummarizedJob(job);
- StringBuffer taskSummary = new StringBuffer();
- taskSummary.append("\nTask Summary");
- taskSummary.append("\n============================");
- taskSummary.append("\nKind\tTotal\t");
- taskSummary.append("Successful\tFailed\tKilled\tStartTime\tFinishTime");
- taskSummary.append("\n");
- taskSummary.append("\nSetup\t").append(ts.totalSetups);
- taskSummary.append("\t").append(ts.numFinishedSetups);
- taskSummary.append("\t\t").append(ts.numFailedSetups);
- taskSummary.append("\t").append(ts.numKilledSetups);
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.setupStarted, 0));
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.setupFinished, ts.setupStarted));
- taskSummary.append("\nMap\t").append(ts.totalMaps);
- taskSummary.append("\t").append(job.getFinishedMaps());
- taskSummary.append("\t\t").append(ts.numFailedMaps);
- taskSummary.append("\t").append(ts.numKilledMaps);
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.mapStarted, 0));
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.mapFinished, ts.mapStarted));
- taskSummary.append("\nReduce\t").append(ts.totalReduces);
- taskSummary.append("\t").append(job.getFinishedReduces());
- taskSummary.append("\t\t").append(ts.numFailedReduces);
- taskSummary.append("\t").append(ts.numKilledReduces);
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.reduceStarted, 0));
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.reduceFinished, ts.reduceStarted));
- taskSummary.append("\nCleanup\t").append(ts.totalCleanups);
- taskSummary.append("\t").append(ts.numFinishedCleanups);
- taskSummary.append("\t\t").append(ts.numFailedCleanups);
- taskSummary.append("\t").append(ts.numKilledCleanups);
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.cleanupStarted, 0));
- taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, ts.cleanupFinished,
- ts.cleanupStarted));
- taskSummary.append("\n============================\n");
- System.out.println(taskSummary.toString());
- }
-
- private void printJobAnalysis() {
- if (!job.getJobStatus().equals
- (JobStatus.getJobRunState(JobStatus.SUCCEEDED))) {
- System.out.println("No Analysis available as job did not finish");
- return;
- }
-
- AnalyzedJob avg = new AnalyzedJob(job);
-
- System.out.println("\nAnalysis");
- System.out.println("=========");
- printAnalysis(avg.getMapTasks(), cMap, "map", avg.getAvgMapTime(), 10);
- printLast(avg.getMapTasks(), "map", cFinishMapRed);
-
- if (avg.getReduceTasks().length > 0) {
- printAnalysis(avg.getReduceTasks(), cShuffle, "shuffle",
- avg.getAvgShuffleTime(), 10);
- printLast(avg.getReduceTasks(), "shuffle", cFinishShuffle);
-
- printAnalysis(avg.getReduceTasks(), cReduce, "reduce",
- avg.getAvgReduceTime(), 10);
- printLast(avg.getReduceTasks(), "reduce", cFinishMapRed);
- }
- System.out.println("=========");
- }
-
- private void printAnalysis(JobHistoryParser.TaskAttemptInfo [] tasks,
- Comparator<JobHistoryParser.TaskAttemptInfo> cmp,
- String taskType,
- long avg,
- int showTasks) {
- Arrays.sort(tasks, cmp);
- JobHistoryParser.TaskAttemptInfo min = tasks[tasks.length-1];
- StringBuffer details = new StringBuffer();
- details.append("\nTime taken by best performing ");
- details.append(taskType).append(" task ");
- details.append(min.getAttemptId().getTaskID().toString()).append(": ");
- if ("map".equals(taskType)) {
- details.append(StringUtils.formatTimeDiff(
- min.getFinishTime(),
- min.getStartTime()));
- } else if ("shuffle".equals(taskType)) {
- details.append(StringUtils.formatTimeDiff(
- min.getShuffleFinishTime(),
- min.getStartTime()));
- } else {
- details.append(StringUtils.formatTimeDiff(
- min.getFinishTime(),
- min.getShuffleFinishTime()));
- }
- details.append("\nAverage time taken by ");
- details.append(taskType).append(" tasks: ");
- details.append(StringUtils.formatTimeDiff(avg, 0));
- details.append("\nWorse performing ");
- details.append(taskType).append(" tasks: ");
- details.append("\nTaskId\t\tTimetaken");
- System.out.println(details.toString());
- for (int i = 0; i < showTasks && i < tasks.length; i++) {
- details.setLength(0);
- details.append(tasks[i].getAttemptId().getTaskID()).append(" ");
- if ("map".equals(taskType)) {
- details.append(StringUtils.formatTimeDiff(
- tasks[i].getFinishTime(),
- tasks[i].getStartTime()));
- } else if ("shuffle".equals(taskType)) {
- details.append(StringUtils.formatTimeDiff(
- tasks[i].getShuffleFinishTime(),
- tasks[i].getStartTime()));
- } else {
- details.append(StringUtils.formatTimeDiff(
- tasks[i].getFinishTime(),
- tasks[i].getShuffleFinishTime()));
- }
- System.out.println(details.toString());
- }
+ public void print() throws IOException {
+ print(System.out);
}
- private void printLast(JobHistoryParser.TaskAttemptInfo [] tasks,
- String taskType,
- Comparator<JobHistoryParser.TaskAttemptInfo> cmp
- ) {
- Arrays.sort(tasks, cFinishMapRed);
- JobHistoryParser.TaskAttemptInfo last = tasks[0];
- StringBuffer lastBuf = new StringBuffer();
- lastBuf.append("The last ").append(taskType);
- lastBuf.append(" task ").append(last.getAttemptId().getTaskID());
- Long finishTime;
- if ("shuffle".equals(taskType)) {
- finishTime = last.getShuffleFinishTime();
- } else {
- finishTime = last.getFinishTime();
- }
- lastBuf.append(" finished at (relative to the Job launch time): ");
- lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
- finishTime, job.getLaunchTime()));
- System.out.println(lastBuf.toString());
- }
-
- private void printTasks(TaskType taskType, String status) {
- Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks();
- StringBuffer header = new StringBuffer();
- header.append("\n").append(status).append(" ");
- header.append(taskType).append(" task list for ").append(jobId);
- header.append("\nTaskId\t\tStartTime\tFinishTime\tError");
- if (TaskType.MAP.equals(taskType)) {
- header.append("\tInputSplits");
- }
- header.append("\n====================================================");
- StringBuffer taskList = new StringBuffer();
- for (JobHistoryParser.TaskInfo task : tasks.values()) {
- if (taskType.equals(task.getTaskType()) &&
- (status.equals(task.getTaskStatus())
- || status.equalsIgnoreCase("ALL"))) {
- taskList.setLength(0);
- taskList.append(task.getTaskId());
- taskList.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, task.getStartTime(), 0));
- taskList.append("\t").append(StringUtils.getFormattedTimeWithDiff(
- dateFormat, task.getFinishTime(),
- task.getStartTime()));
- taskList.append("\t").append(task.getError());
- if (TaskType.MAP.equals(taskType)) {
- taskList.append("\t").append(task.getSplitLocations());
- }
- if (taskList != null) {
- System.out.println(header.toString());
- System.out.println(taskList.toString());
- }
- }
- }
- }
-
- private void printFailedAttempts(FilteredJob filteredJob) {
- Map<String, Set<TaskID>> badNodes = filteredJob.getFilteredMap();
- StringBuffer attempts = new StringBuffer();
- if (badNodes.size() > 0) {
- attempts.append("\n").append(filteredJob.getFilter());
- attempts.append(" task attempts by nodes");
- attempts.append("\nHostname\tFailedTasks");
- attempts.append("\n===============================");
- System.out.println(attempts.toString());
- for (Map.Entry<String,
- Set<TaskID>> entry : badNodes.entrySet()) {
- String node = entry.getKey();
- Set<TaskID> failedTasks = entry.getValue();
- attempts.setLength(0);
- attempts.append(node).append("\t");
- for (TaskID t : failedTasks) {
- attempts.append(t).append(", ");
- }
- System.out.println(attempts.toString());
- }
- }
+ /**
+ * Print the job/task/attempt summary information to the PrintStream.
+ * @param ps The PrintStream to print to
+ * @throws IOException when there is a problem printing the history
+ */
+ public void print(PrintStream ps) throws IOException {
+ jhvp.print(ps);
}
/**
- * Return the TaskLogsUrl of a particular TaskAttempt
+ * Return the TaskLogsUrl of a particular TaskAttempt.
*
- * @param attempt
+ * @param attempt info about the task attempt
* @return the taskLogsUrl. null if http-port or tracker-name or
* task-attempt-id are unavailable.
*/
@@ -480,58 +156,6 @@ public class HistoryViewer {
attempt.getAttemptId().toString());
}
- private Comparator<JobHistoryParser.TaskAttemptInfo> cMap =
- new Comparator<JobHistoryParser.TaskAttemptInfo>() {
- public int compare(JobHistoryParser.TaskAttemptInfo t1,
- JobHistoryParser.TaskAttemptInfo t2) {
- long l1 = t1.getFinishTime() - t1.getStartTime();
- long l2 = t2.getFinishTime() - t2.getStartTime();
- return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1));
- }
- };
-
- private Comparator<JobHistoryParser.TaskAttemptInfo> cShuffle =
- new Comparator<JobHistoryParser.TaskAttemptInfo>() {
- public int compare(JobHistoryParser.TaskAttemptInfo t1,
- JobHistoryParser.TaskAttemptInfo t2) {
- long l1 = t1.getShuffleFinishTime() - t1.getStartTime();
- long l2 = t2.getShuffleFinishTime() - t2.getStartTime();
- return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1));
- }
- };
-
- private Comparator<JobHistoryParser.TaskAttemptInfo> cFinishShuffle =
- new Comparator<JobHistoryParser.TaskAttemptInfo>() {
- public int compare(JobHistoryParser.TaskAttemptInfo t1,
- JobHistoryParser.TaskAttemptInfo t2) {
- long l1 = t1.getShuffleFinishTime();
- long l2 = t2.getShuffleFinishTime();
- return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1));
- }
- };
-
- private Comparator<JobHistoryParser.TaskAttemptInfo> cFinishMapRed =
- new Comparator<JobHistoryParser.TaskAttemptInfo>() {
- public int compare(JobHistoryParser.TaskAttemptInfo t1,
- JobHistoryParser.TaskAttemptInfo t2) {
- long l1 = t1.getFinishTime();
- long l2 = t2.getFinishTime();
- return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1));
- }
- };
-
- private Comparator<JobHistoryParser.TaskAttemptInfo> cReduce =
- new Comparator<JobHistoryParser.TaskAttemptInfo>() {
- public int compare(JobHistoryParser.TaskAttemptInfo t1,
- JobHistoryParser.TaskAttemptInfo t2) {
- long l1 = t1.getFinishTime() -
- t1.getShuffleFinishTime();
- long l2 = t2.getFinishTime() -
- t2.getShuffleFinishTime();
- return (l2 < l1 ? -1 : (l2 == l1 ? 0 : 1));
- }
- };
-
/**
* Utility class used the summarize the job.
* Used by HistoryViewer and the JobHistory UI.
@@ -694,7 +318,6 @@ public class HistoryViewer {
* Utility class used while analyzing the job.
* Used by HistoryViewer and the JobHistory UI.
*/
-
public static class AnalyzedJob {
private long avgMapTime;
private long avgReduceTime;
@@ -765,7 +388,6 @@ public class HistoryViewer {
/**
* Utility to filter out events based on the task status
- *
*/
public static class FilteredJob {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewerPrinter.java
new file mode 100644
index 0000000..406b439
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewerPrinter.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * Used by the {@link HistoryViewer} to print job history in different formats.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface HistoryViewerPrinter {
+
+ /**
+ * Print out the Job History to the given {@link PrintStream}.
+ * @param ps the {@link PrintStream} to print to
+ * @throws IOException when a problem occurs while printing
+ */
+ void print(PrintStream ps) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
new file mode 100644
index 0000000..30903af
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
@@ -0,0 +1,471 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.DecimalFormat;
+import java.text.Format;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+
+/**
+ * Used by the {@link HistoryViewer} to print job history in a human-readable
+ * format.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter {
+
+ private JobHistoryParser.JobInfo job;
+ private final SimpleDateFormat dateFormat;
+ private boolean printAll;
+ private String scheme;
+
+ HumanReadableHistoryViewerPrinter(JobHistoryParser.JobInfo job,
+ boolean printAll, String scheme) {
+ this.job = job;
+ this.printAll = printAll;
+ this.scheme = scheme;
+ this.dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss");
+ }
+
+ HumanReadableHistoryViewerPrinter(JobHistoryParser.JobInfo job,
+ boolean printAll, String scheme,
+ TimeZone tz) {
+ this(job, printAll, scheme);
+ this.dateFormat.setTimeZone(tz);
+ }
+
+ /**
+ * Print out the Job History to the given {@link PrintStream} in a
+ * human-readable format.
+ * @param ps the {@link PrintStream} to print to
+ * @throws IOException when a problem occurs while printing
+ */
+ @Override
+ public void print(PrintStream ps) throws IOException {
+ printJobDetails(ps);
+ printTaskSummary(ps);
+ printJobAnalysis(ps);
+ printTasks(ps, TaskType.JOB_SETUP, TaskStatus.State.FAILED.toString());
+ printTasks(ps, TaskType.JOB_SETUP, TaskStatus.State.KILLED.toString());
+ printTasks(ps, TaskType.MAP, TaskStatus.State.FAILED.toString());
+ printTasks(ps, TaskType.MAP, TaskStatus.State.KILLED.toString());
+ printTasks(ps, TaskType.REDUCE, TaskStatus.State.FAILED.toString());
+ printTasks(ps, TaskType.REDUCE, TaskStatus.State.KILLED.toString());
+ printTasks(ps, TaskType.JOB_CLEANUP, TaskStatus.State.FAILED.toString());
+ printTasks(ps, TaskType.JOB_CLEANUP,
+ JobStatus.getJobRunState(JobStatus.KILLED));
+ if (printAll) {
+ printTasks(ps, TaskType.JOB_SETUP, TaskStatus.State.SUCCEEDED.toString());
+ printTasks(ps, TaskType.MAP, TaskStatus.State.SUCCEEDED.toString());
+ printTasks(ps, TaskType.REDUCE, TaskStatus.State.SUCCEEDED.toString());
+ printTasks(ps, TaskType.JOB_CLEANUP,
+ TaskStatus.State.SUCCEEDED.toString());
+ printAllTaskAttempts(ps, TaskType.JOB_SETUP);
+ printAllTaskAttempts(ps, TaskType.MAP);
+ printAllTaskAttempts(ps, TaskType.REDUCE);
+ printAllTaskAttempts(ps, TaskType.JOB_CLEANUP);
+ }
+
+ HistoryViewer.FilteredJob filter = new HistoryViewer.FilteredJob(job,
+ TaskStatus.State.FAILED.toString());
+ printFailedAttempts(ps, filter);
+
+ filter = new HistoryViewer.FilteredJob(job,
+ TaskStatus.State.KILLED.toString());
+ printFailedAttempts(ps, filter);
+ }
+
+ private void printJobDetails(PrintStream ps) {
+ StringBuilder jobDetails = new StringBuilder();
+ jobDetails.append("\nHadoop job: ").append(job.getJobId());
+ jobDetails.append("\n=====================================");
+ jobDetails.append("\nUser: ").append(job.getUsername());
+ jobDetails.append("\nJobName: ").append(job.getJobname());
+ jobDetails.append("\nJobConf: ").append(job.getJobConfPath());
+ jobDetails.append("\nSubmitted At: ").append(StringUtils.
+ getFormattedTimeWithDiff(dateFormat,
+ job.getSubmitTime(), 0));
+ jobDetails.append("\nLaunched At: ").append(StringUtils.
+ getFormattedTimeWithDiff(dateFormat,
+ job.getLaunchTime(),
+ job.getSubmitTime()));
+ jobDetails.append("\nFinished At: ").append(StringUtils.
+ getFormattedTimeWithDiff(dateFormat,
+ job.getFinishTime(),
+ job.getLaunchTime()));
+ jobDetails.append("\nStatus: ").append(((job.getJobStatus() == null) ?
+ "Incomplete" :job.getJobStatus()));
+ printJobCounters(jobDetails, job.getTotalCounters(), job.getMapCounters(),
+ job.getReduceCounters());
+ jobDetails.append("\n");
+ jobDetails.append("\n=====================================");
+ ps.println(jobDetails);
+ }
+
+ private void printJobCounters(StringBuilder buff, Counters totalCounters,
+ Counters mapCounters, Counters reduceCounters) {
+ // Killed jobs might not have counters
+ if (totalCounters != null) {
+ buff.append("\nCounters: \n\n");
+ buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|",
+ "Group Name",
+ "Counter name",
+ "Map Value",
+ "Reduce Value",
+ "Total Value"));
+ buff.append("\n------------------------------------------" +
+ "---------------------------------------------");
+ for (String groupName : totalCounters.getGroupNames()) {
+ CounterGroup totalGroup = totalCounters.getGroup(groupName);
+ CounterGroup mapGroup = mapCounters.getGroup(groupName);
+ CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
+
+ Format decimal = new DecimalFormat();
+ Iterator<Counter> ctrItr =
+ totalGroup.iterator();
+ while (ctrItr.hasNext()) {
+ org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
+ String name = counter.getName();
+ String mapValue =
+ decimal.format(mapGroup.findCounter(name).getValue());
+ String reduceValue =
+ decimal.format(reduceGroup.findCounter(name).getValue());
+ String totalValue =
+ decimal.format(counter.getValue());
+
+ buff.append(
+ String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s",
+ totalGroup.getDisplayName(),
+ counter.getDisplayName(),
+ mapValue, reduceValue, totalValue));
+ }
+ }
+ }
+ }
+
+ private void printAllTaskAttempts(PrintStream ps, TaskType taskType) {
+ Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks();
+ StringBuilder taskList = new StringBuilder();
+ taskList.append("\n").append(taskType);
+ taskList.append(" task list for ").append(job.getJobId());
+ taskList.append("\nTaskId\t\tStartTime");
+ if (TaskType.REDUCE.equals(taskType)) {
+ taskList.append("\tShuffleFinished\tSortFinished");
+ }
+ taskList.append("\tFinishTime\tHostName\tError\tTaskLogs");
+ taskList.append("\n====================================================");
+ ps.println(taskList.toString());
+ for (JobHistoryParser.TaskInfo task : tasks.values()) {
+ for (JobHistoryParser.TaskAttemptInfo attempt :
+ task.getAllTaskAttempts().values()) {
+ if (taskType.equals(task.getTaskType())){
+ taskList.setLength(0);
+ taskList.append(attempt.getAttemptId()).append("\t");
+ taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
+ attempt.getStartTime(), 0)).append("\t");
+ if (TaskType.REDUCE.equals(taskType)) {
+ taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
+ attempt.getShuffleFinishTime(),
+ attempt.getStartTime()));
+ taskList.append("\t");
+ taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
+ attempt.getSortFinishTime(),
+ attempt.getShuffleFinishTime()));
+ }
+ taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
+ attempt.getFinishTime(),
+ attempt.getStartTime()));
+ taskList.append("\t");
+ taskList.append(attempt.getHostname()).append("\t");
+ taskList.append(attempt.getError());
+ String taskLogsUrl = HistoryViewer.getTaskLogsUrl(scheme, attempt);
+ taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a");
+ ps.println(taskList);
+ }
+ }
+ }
+ }
+
+ private void printTaskSummary(PrintStream ps) {
+ HistoryViewer.SummarizedJob ts = new HistoryViewer.SummarizedJob(job);
+ StringBuilder taskSummary = new StringBuilder();
+ taskSummary.append("\nTask Summary");
+ taskSummary.append("\n============================");
+ taskSummary.append("\nKind\tTotal\t");
+ taskSummary.append("Successful\tFailed\tKilled\tStartTime\tFinishTime");
+ taskSummary.append("\n");
+ taskSummary.append("\nSetup\t").append(ts.totalSetups);
+ taskSummary.append("\t").append(ts.numFinishedSetups);
+ taskSummary.append("\t\t").append(ts.numFailedSetups);
+ taskSummary.append("\t").append(ts.numKilledSetups);
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.setupStarted, 0));
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.setupFinished, ts.setupStarted));
+ taskSummary.append("\nMap\t").append(ts.totalMaps);
+ taskSummary.append("\t").append(job.getFinishedMaps());
+ taskSummary.append("\t\t").append(ts.numFailedMaps);
+ taskSummary.append("\t").append(ts.numKilledMaps);
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.mapStarted, 0));
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.mapFinished, ts.mapStarted));
+ taskSummary.append("\nReduce\t").append(ts.totalReduces);
+ taskSummary.append("\t").append(job.getFinishedReduces());
+ taskSummary.append("\t\t").append(ts.numFailedReduces);
+ taskSummary.append("\t").append(ts.numKilledReduces);
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.reduceStarted, 0));
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.reduceFinished, ts.reduceStarted));
+ taskSummary.append("\nCleanup\t").append(ts.totalCleanups);
+ taskSummary.append("\t").append(ts.numFinishedCleanups);
+ taskSummary.append("\t\t").append(ts.numFailedCleanups);
+ taskSummary.append("\t").append(ts.numKilledCleanups);
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.cleanupStarted, 0));
+ taskSummary.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, ts.cleanupFinished,
+ ts.cleanupStarted));
+ taskSummary.append("\n============================\n");
+ ps.println(taskSummary);
+ }
+
+ private void printJobAnalysis(PrintStream ps) {
+ if (job.getJobStatus().equals(
+ JobStatus.getJobRunState(JobStatus.SUCCEEDED))) {
+ HistoryViewer.AnalyzedJob avg = new HistoryViewer.AnalyzedJob(job);
+
+ ps.println("\nAnalysis");
+ ps.println("=========");
+ printAnalysis(ps, avg.getMapTasks(), cMap, "map", avg.getAvgMapTime(),
+ 10);
+ printLast(ps, avg.getMapTasks(), "map", cFinishMapRed);
+
+ if (avg.getReduceTasks().length > 0) {
+ printAnalysis(ps, avg.getReduceTasks(), cShuffle, "shuffle",
+ avg.getAvgShuffleTime(), 10);
+ printLast(ps, avg.getReduceTasks(), "shuffle", cFinishShuffle);
+
+ printAnalysis(ps, avg.getReduceTasks(), cReduce, "reduce",
+ avg.getAvgReduceTime(), 10);
+ printLast(ps, avg.getReduceTasks(), "reduce", cFinishMapRed);
+ }
+ ps.println("=========");
+ } else {
+ ps.println("No Analysis available as job did not finish");
+ }
+ }
+
+ protected void printAnalysis(PrintStream ps,
+ JobHistoryParser.TaskAttemptInfo[] tasks,
+ Comparator<JobHistoryParser.TaskAttemptInfo> cmp,
+ String taskType, long avg, int showTasks) {
+ Arrays.sort(tasks, cmp);
+ JobHistoryParser.TaskAttemptInfo min = tasks[tasks.length-1];
+ StringBuilder details = new StringBuilder();
+ details.append("\nTime taken by best performing ");
+ details.append(taskType).append(" task ");
+ details.append(min.getAttemptId().getTaskID().toString()).append(": ");
+ if ("map".equals(taskType)) {
+ details.append(StringUtils.formatTimeDiff(
+ min.getFinishTime(),
+ min.getStartTime()));
+ } else if ("shuffle".equals(taskType)) {
+ details.append(StringUtils.formatTimeDiff(
+ min.getShuffleFinishTime(),
+ min.getStartTime()));
+ } else {
+ details.append(StringUtils.formatTimeDiff(
+ min.getFinishTime(),
+ min.getShuffleFinishTime()));
+ }
+ details.append("\nAverage time taken by ");
+ details.append(taskType).append(" tasks: ");
+ details.append(StringUtils.formatTimeDiff(avg, 0));
+ details.append("\nWorse performing ");
+ details.append(taskType).append(" tasks: ");
+ details.append("\nTaskId\t\tTimetaken");
+ ps.println(details);
+ for (int i = 0; i < showTasks && i < tasks.length; i++) {
+ details.setLength(0);
+ details.append(tasks[i].getAttemptId().getTaskID()).append(" ");
+ if ("map".equals(taskType)) {
+ details.append(StringUtils.formatTimeDiff(
+ tasks[i].getFinishTime(),
+ tasks[i].getStartTime()));
+ } else if ("shuffle".equals(taskType)) {
+ details.append(StringUtils.formatTimeDiff(
+ tasks[i].getShuffleFinishTime(),
+ tasks[i].getStartTime()));
+ } else {
+ details.append(StringUtils.formatTimeDiff(
+ tasks[i].getFinishTime(),
+ tasks[i].getShuffleFinishTime()));
+ }
+ ps.println(details);
+ }
+ }
+
+ protected void printLast(PrintStream ps,
+ JobHistoryParser.TaskAttemptInfo[] tasks, String taskType,
+ Comparator<JobHistoryParser.TaskAttemptInfo> cmp) {
+ Arrays.sort(tasks, cFinishMapRed);
+ JobHistoryParser.TaskAttemptInfo last = tasks[0];
+ StringBuilder lastBuf = new StringBuilder();
+ lastBuf.append("The last ").append(taskType);
+ lastBuf.append(" task ").append(last.getAttemptId().getTaskID());
+ Long finishTime;
+ if ("shuffle".equals(taskType)) {
+ finishTime = last.getShuffleFinishTime();
+ } else {
+ finishTime = last.getFinishTime();
+ }
+ lastBuf.append(" finished at (relative to the Job launch time): ");
+ lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat,
+ finishTime, job.getLaunchTime()));
+ ps.println(lastBuf);
+ }
+
+ private void printTasks(PrintStream ps, TaskType taskType, String status) {
+ Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks();
+ StringBuilder header = new StringBuilder();
+ header.append("\n").append(status).append(" ");
+ header.append(taskType).append(" task list for ")
+ .append(job.getJobId().toString());
+ header.append("\nTaskId\t\tStartTime\tFinishTime\tError");
+ if (TaskType.MAP.equals(taskType)) {
+ header.append("\tInputSplits");
+ }
+ header.append("\n====================================================");
+ StringBuilder taskList = new StringBuilder();
+ for (JobHistoryParser.TaskInfo task : tasks.values()) {
+ if (taskType.equals(task.getTaskType()) &&
+ (status.equals(task.getTaskStatus())
+ || status.equalsIgnoreCase("ALL"))) {
+ taskList.setLength(0);
+ taskList.append(task.getTaskId());
+ taskList.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, task.getStartTime(), 0));
+ taskList.append("\t").append(StringUtils.getFormattedTimeWithDiff(
+ dateFormat, task.getFinishTime(),
+ task.getStartTime()));
+ taskList.append("\t").append(task.getError());
+ if (TaskType.MAP.equals(taskType)) {
+ taskList.append("\t").append(task.getSplitLocations());
+ }
+ if (taskList != null) {
+ ps.println(header);
+ ps.println(taskList);
+ }
+ }
+ }
+ }
+
+ private void printFailedAttempts(PrintStream ps,
+ HistoryViewer.FilteredJob filteredJob) {
+ Map<String, Set<TaskID>> badNodes = filteredJob.getFilteredMap();
+ StringBuilder attempts = new StringBuilder();
+ if (badNodes.size() > 0) {
+ attempts.append("\n").append(filteredJob.getFilter());
+ attempts.append(" task attempts by nodes");
+ attempts.append("\nHostname\tFailedTasks");
+ attempts.append("\n===============================");
+ ps.println(attempts);
+ for (Map.Entry<String, Set<TaskID>> entry : badNodes.entrySet()) {
+ String node = entry.getKey();
+ Set<TaskID> failedTasks = entry.getValue();
+ attempts.setLength(0);
+ attempts.append(node).append("\t");
+ for (TaskID t : failedTasks) {
+ attempts.append(t).append(", ");
+ }
+ ps.println(attempts);
+ }
+ }
+ }
+
+ private static Comparator<JobHistoryParser.TaskAttemptInfo> cMap =
+ new Comparator<JobHistoryParser.TaskAttemptInfo>() {
+ public int compare(JobHistoryParser.TaskAttemptInfo t1,
+ JobHistoryParser.TaskAttemptInfo t2) {
+ long l1 = t1.getFinishTime() - t1.getStartTime();
+ long l2 = t2.getFinishTime() - t2.getStartTime();
+ return Long.compare(l2, l1);
+ }
+ };
+
+ private static Comparator<JobHistoryParser.TaskAttemptInfo> cShuffle =
+ new Comparator<JobHistoryParser.TaskAttemptInfo>() {
+ public int compare(JobHistoryParser.TaskAttemptInfo t1,
+ JobHistoryParser.TaskAttemptInfo t2) {
+ long l1 = t1.getShuffleFinishTime() - t1.getStartTime();
+ long l2 = t2.getShuffleFinishTime() - t2.getStartTime();
+ return Long.compare(l2, l1);
+ }
+ };
+
+ private static Comparator<JobHistoryParser.TaskAttemptInfo> cFinishShuffle =
+ new Comparator<JobHistoryParser.TaskAttemptInfo>() {
+ public int compare(JobHistoryParser.TaskAttemptInfo t1,
+ JobHistoryParser.TaskAttemptInfo t2) {
+ long l1 = t1.getShuffleFinishTime();
+ long l2 = t2.getShuffleFinishTime();
+ return Long.compare(l2, l1);
+ }
+ };
+
+ private static Comparator<JobHistoryParser.TaskAttemptInfo> cFinishMapRed =
+ new Comparator<JobHistoryParser.TaskAttemptInfo>() {
+ public int compare(JobHistoryParser.TaskAttemptInfo t1,
+ JobHistoryParser.TaskAttemptInfo t2) {
+ long l1 = t1.getFinishTime();
+ long l2 = t2.getFinishTime();
+ return Long.compare(l2, l1);
+ }
+ };
+
+ private static Comparator<JobHistoryParser.TaskAttemptInfo> cReduce =
+ new Comparator<JobHistoryParser.TaskAttemptInfo>() {
+ public int compare(JobHistoryParser.TaskAttemptInfo t1,
+ JobHistoryParser.TaskAttemptInfo t2) {
+ long l1 = t1.getFinishTime() - t1.getShuffleFinishTime();
+ long l2 = t2.getFinishTime() - t2.getShuffleFinishTime();
+ return Long.compare(l2, l1);
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
new file mode 100644
index 0000000..456dcf7
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.io.Writer;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Used by the {@link HistoryViewer} to print job history in a machine-readable
+ * JSON format.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class JSONHistoryViewerPrinter implements HistoryViewerPrinter {
+
+ private JobHistoryParser.JobInfo job;
+ private boolean printAll;
+ private String scheme;
+ private JSONObject json;
+
+ JSONHistoryViewerPrinter(JobHistoryParser.JobInfo job, boolean printAll,
+ String scheme) {
+ this.job = job;
+ this.printAll = printAll;
+ this.scheme = scheme;
+ }
+
+ /**
+ * Print out the Job History to the given {@link PrintStream} in a
+ * machine-readable JSON format.
+ * @param ps the {@link PrintStream} to print to
+ * @throws IOException when a problem occurs while printing
+ */
+ @Override
+ public void print(PrintStream ps) throws IOException {
+ json = new JSONObject();
+
+ Writer writer = null;
+ try {
+ printJobDetails();
+ printTaskSummary();
+ printTasks();
+
+ writer = new OutputStreamWriter(ps, "UTF-8");
+ json.write(writer);
+ writer.flush();
+ } catch (JSONException je) {
+ throw new IOException("Failure parsing JSON document: " + je.getMessage(),
+ je);
+ } finally {
+ if (writer != null) {
+ writer.close();
+ }
+ }
+ }
+
+ private void printJobDetails() throws JSONException {
+ json.put("hadoopJob", job.getJobId().toString());
+ json.put("user", job.getUsername());
+ json.put("jobName", job.getJobname());
+ json.put("jobConf", job.getJobConfPath());
+ json.put("submittedAt", job.getSubmitTime());
+ json.put("launchedAt", job.getLaunchTime());
+ json.put("finishedAt", job.getFinishTime());
+ json.put("status", ((job.getJobStatus() == null) ?
+ "Incomplete" :job.getJobStatus()));
+ printJobCounters(job.getTotalCounters(), job.getMapCounters(),
+ job.getReduceCounters());
+ }
+
+ private void printJobCounters(Counters totalCounters, Counters mapCounters,
+ Counters reduceCounters) throws JSONException {
+ // Killed jobs might not have counters
+ if (totalCounters != null) {
+ JSONObject jGroups = new JSONObject();
+ for (String groupName : totalCounters.getGroupNames()) {
+ CounterGroup totalGroup = totalCounters.getGroup(groupName);
+ CounterGroup mapGroup = mapCounters.getGroup(groupName);
+ CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
+
+ Iterator<Counter> ctrItr =
+ totalGroup.iterator();
+ JSONArray jGroup = new JSONArray();
+ while (ctrItr.hasNext()) {
+ JSONObject jCounter = new JSONObject();
+ org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
+ String name = counter.getName();
+ long mapValue = mapGroup.findCounter(name).getValue();
+ long reduceValue = reduceGroup.findCounter(name).getValue();
+ long totalValue = counter.getValue();
+
+ jCounter.put("counterName", name);
+ jCounter.put("mapValue", mapValue);
+ jCounter.put("reduceValue", reduceValue);
+ jCounter.put("totalValue", totalValue);
+ jGroup.put(jCounter);
+ }
+ jGroups.put(fixGroupNameForShuffleErrors(totalGroup.getName()), jGroup);
+ }
+ json.put("counters", jGroups);
+ }
+ }
+
+ private void printTaskSummary() throws JSONException {
+ HistoryViewer.SummarizedJob ts = new HistoryViewer.SummarizedJob(job);
+ JSONObject jSums = new JSONObject();
+ JSONObject jSumSetup = new JSONObject();
+ jSumSetup.put("total", ts.totalSetups);
+ jSumSetup.put("successful", ts.numFinishedSetups);
+ jSumSetup.put("failed", ts.numFailedSetups);
+ jSumSetup.put("killed", ts.numKilledSetups);
+ jSumSetup.put("startTime", ts.setupStarted);
+ jSumSetup.put("finishTime", ts.setupFinished);
+ jSums.put("setup", jSumSetup);
+ JSONObject jSumMap = new JSONObject();
+ jSumMap.put("total", ts.totalMaps);
+ jSumMap.put("successful", job.getFinishedMaps());
+ jSumMap.put("failed", ts.numFailedMaps);
+ jSumMap.put("killed", ts.numKilledMaps);
+ jSumMap.put("startTime", ts.mapStarted);
+ jSumMap.put("finishTime", ts.mapFinished);
+ jSums.put("map", jSumMap);
+ JSONObject jSumReduce = new JSONObject();
+ jSumReduce.put("total", ts.totalReduces);
+ jSumReduce.put("successful", job.getFinishedReduces());
+ jSumReduce.put("failed", ts.numFailedReduces);
+ jSumReduce.put("killed", ts.numKilledReduces);
+ jSumReduce.put("startTime", ts.reduceStarted);
+ jSumReduce.put("finishTime", ts.reduceFinished);
+ jSums.put("reduce", jSumReduce);
+ JSONObject jSumCleanup = new JSONObject();
+ jSumCleanup.put("total", ts.totalCleanups);
+ jSumCleanup.put("successful", ts.numFinishedCleanups);
+ jSumCleanup.put("failed", ts.numFailedCleanups);
+ jSumCleanup.put("killed", ts.numKilledCleanups);
+ jSumCleanup.put("startTime", ts.cleanupStarted);
+ jSumCleanup.put("finishTime", ts.cleanupFinished);
+ jSums.put("cleanup", jSumCleanup);
+ json.put("taskSummary", jSums);
+ }
+
+ private void printTasks() throws JSONException {
+ Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks();
+ JSONArray jTasks = new JSONArray();
+ for (JobHistoryParser.TaskInfo task : tasks.values()) {
+ if (!task.getTaskType().equals(TaskType.TASK_CLEANUP) &&
+ ((printAll && task.getTaskStatus().equals(
+ TaskStatus.State.SUCCEEDED.toString()))
+ || task.getTaskStatus().equals(TaskStatus.State.KILLED.toString())
+ || task.getTaskStatus().equals(TaskStatus.State.FAILED.toString()))) {
+ JSONObject jTask = new JSONObject();
+ jTask.put("taskId", task.getTaskId().toString());
+ jTask.put("type", task.getTaskType().toString());
+ jTask.put("status", task.getTaskStatus());
+ jTask.put("startTime", task.getStartTime());
+ jTask.put("finishTime", task.getFinishTime());
+ if (!task.getError().isEmpty()) {
+ jTask.put("error", task.getError());
+ }
+ if (task.getTaskType().equals(TaskType.MAP)) {
+ jTask.put("inputSplits", task.getSplitLocations());
+ }
+ if (printAll) {
+ printTaskCounters(jTask, task.getCounters());
+ JSONObject jAtt = new JSONObject();
+ for (JobHistoryParser.TaskAttemptInfo attempt :
+ task.getAllTaskAttempts().values()) {
+ jAtt.put("attemptId", attempt.getAttemptId());
+ jAtt.put("startTime", attempt.getStartTime());
+ if (task.getTaskType().equals(TaskType.REDUCE)) {
+ jAtt.put("shuffleFinished", attempt.getShuffleFinishTime());
+ jAtt.put("sortFinished", attempt.getSortFinishTime());
+ }
+ jAtt.put("finishTime", attempt.getFinishTime());
+ jAtt.put("hostName", attempt.getHostname());
+ if (!attempt.getError().isEmpty()) {
+ jAtt.put("error", task.getError());
+ }
+ String taskLogsUrl = HistoryViewer.getTaskLogsUrl(scheme, attempt);
+ if (taskLogsUrl != null) {
+ jAtt.put("taskLogs", taskLogsUrl);
+ }
+ }
+ jTask.put("attempts", jAtt);
+ }
+ jTasks.put(jTask);
+ }
+ json.put("tasks", jTasks);
+ }
+ }
+
+ private void printTaskCounters(JSONObject jTask, Counters taskCounters)
+ throws JSONException {
+ // Killed tasks might not have counters
+ if (taskCounters != null) {
+ JSONObject jGroups = new JSONObject();
+ for (String groupName : taskCounters.getGroupNames()) {
+ CounterGroup group = taskCounters.getGroup(groupName);
+
+ Iterator<Counter> ctrItr = group.iterator();
+ JSONArray jGroup = new JSONArray();
+ while (ctrItr.hasNext()) {
+ JSONObject jCounter = new JSONObject();
+ org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
+
+ jCounter.put("counterName", counter.getName());
+ jCounter.put("value", counter.getValue());
+ jGroup.put(jCounter);
+ }
+ jGroups.put(fixGroupNameForShuffleErrors(group.getName()), jGroup);
+ }
+ jTask.put("counters", jGroups);
+ }
+ }
+
+ private String fixGroupNameForShuffleErrors(String name) {
+ String retName = name;
+
+ if (name.equals("Shuffle Errors")) {
+ retName = "org.apache.hadoop.mapreduce.task.reduce.Fetcher.ShuffleErrors";
+ }
+
+ return retName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
index db87d9d..ea9c1c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
@@ -17,8 +17,12 @@
*/
package org.apache.hadoop.mapreduce.tools;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
@@ -92,6 +96,8 @@ public class CLI extends Configured implements Tool {
String jobid = null;
String taskid = null;
String historyFile = null;
+ String historyOutFile = null;
+ String historyOutFormat = HistoryViewer.HUMAN_FORMAT;
String counterGroupName = null;
String counterName = null;
JobPriority jp = null;
@@ -173,16 +179,51 @@ public class CLI extends Configured implements Tool {
nEvents = Integer.parseInt(argv[3]);
listEvents = true;
} else if ("-history".equals(cmd)) {
- if (argv.length != 2 && !(argv.length == 3 && "all".equals(argv[1]))) {
- displayUsage(cmd);
- return exitCode;
- }
viewHistory = true;
- if (argv.length == 3 && "all".equals(argv[1])) {
+ if (argv.length < 2 || argv.length > 7) {
+ displayUsage(cmd);
+ return exitCode;
+ }
+
+ // Some arguments are optional while others are not, and some require
+ // second arguments. Due to this, the indexing can vary depending on
+ // what's specified and what's left out, as summarized in the below table:
+ // [all] <jobHistoryFile> [-outfile <file>] [-format <human|json>]
+ // 1 2 3 4 5 6
+ // 1 2 3 4
+ // 1 2 3 4
+ // 1 2
+ // 1 2 3 4 5
+ // 1 2 3
+ // 1 2 3
+ // 1
+
+ // "all" is optional, but comes first if specified
+ int index = 1;
+ if ("all".equals(argv[index])) {
+ index++;
viewAllHistory = true;
- historyFile = argv[2];
- } else {
- historyFile = argv[1];
+ if (argv.length == 2) {
+ displayUsage(cmd);
+ return exitCode;
+ }
+ }
+ // Get the job history file argument
+ historyFile = argv[index++];
+ // "-outfile" is optional, but if specified requires a second argument
+ if (argv.length > index + 1 && "-outfile".equals(argv[index])) {
+ index++;
+ historyOutFile = argv[index++];
+ }
+ // "-format" is optional, but if specified required a second argument
+ if (argv.length > index + 1 && "-format".equals(argv[index])) {
+ index++;
+ historyOutFormat = argv[index++];
+ }
+ // Check for any extra arguments that don't belong here
+ if (argv.length > index) {
+ displayUsage(cmd);
+ return exitCode;
}
} else if ("-list".equals(cmd)) {
if (argv.length != 1 && !(argv.length == 2 && "all".equals(argv[1]))) {
@@ -338,7 +379,8 @@ public class CLI extends Configured implements Tool {
exitCode = 0;
}
} else if (viewHistory) {
- viewHistory(historyFile, viewAllHistory);
+ viewHistory(historyFile, viewAllHistory, historyOutFile,
+ historyOutFormat);
exitCode = 0;
} else if (listEvents) {
listEvents(getJob(JobID.forName(jobid)), fromEvent, nEvents);
@@ -451,7 +493,8 @@ public class CLI extends Configured implements Tool {
System.err.println(prefix + "[" + cmd +
" <job-id> <from-event-#> <#-of-events>]. Event #s start from 1.");
} else if ("-history".equals(cmd)) {
- System.err.println(prefix + "[" + cmd + " <jobHistoryFile>]");
+ System.err.println(prefix + "[" + cmd +
+ " [all] <jobHistoryFile> [-outfile <file>] [-format <human|json>]]");
} else if ("-list".equals(cmd)) {
System.err.println(prefix + "[" + cmd + " [all]]");
} else if ("-kill-task".equals(cmd) || "-fail-task".equals(cmd)) {
@@ -484,7 +527,8 @@ public class CLI extends Configured implements Tool {
"Valid values for priorities are: " + jobPriorityValues +
". In addition to this, integers also can be used." + "%n");
System.err.printf("\t[-events <job-id> <from-event-#> <#-of-events>]%n");
- System.err.printf("\t[-history <jobHistoryFile>]%n");
+ System.err.printf("\t[-history [all] <jobHistoryFile> [-outfile <file>]" +
+ " [-format <human|json>]]%n");
System.err.printf("\t[-list [all]]%n");
System.err.printf("\t[-list-active-trackers]%n");
System.err.printf("\t[-list-blacklisted-trackers]%n");
@@ -499,11 +543,16 @@ public class CLI extends Configured implements Tool {
}
}
- private void viewHistory(String historyFile, boolean all)
- throws IOException {
+ private void viewHistory(String historyFile, boolean all,
+ String historyOutFile, String format) throws IOException {
HistoryViewer historyViewer = new HistoryViewer(historyFile,
- getConf(), all);
- historyViewer.print();
+ getConf(), all, format);
+ PrintStream ps = System.out;
+ if (historyOutFile != null) {
+ ps = new PrintStream(new BufferedOutputStream(new FileOutputStream(
+ new File(historyOutFile))), true, "UTF-8");
+ }
+ historyViewer.print(ps);
}
protected long getCounter(Counters counters, String counterGroupName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
index 7d6f5ff..dda2dde 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
@@ -81,7 +81,7 @@ Copy file or directories recursively. More information can be found at
Command to interact with Map Reduce Jobs.
-Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobOutputDir>] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
+Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -90,7 +90,7 @@ Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id
| -counter *job-id* *group-name* *counter-name* | Prints the counter value. |
| -kill *job-id* | Kills the job. |
| -events *job-id* *from-event-\#* *\#-of-events* | Prints the events' details received by jobtracker for the given range. |
-| -history [all]*jobOutputDir* | Prints job details, failed and killed tip details. More details about the job such as successful tasks and task attempts made for each task can be viewed by specifying the [all] option. |
+| -history [all] *jobHistoryFile* [-outfile *file*] [-format *human|json*] | Prints job details, failed and killed task details. More details about the job such as successful tasks, task attempts made for each task, task counters, etc can be viewed by specifying the [all] option. An optional file output path (instead of stdout) can be specified. The format defaults to human-readable but can also be changed to JSON with the [-format] option. |
| -list [all] | Displays jobs which are yet to complete. `-list all` displays all jobs. |
| -kill-task *task-id* | Kills the task. Killed tasks are NOT counted against failed attempts. |
| -fail-task *task-id* | Fails the task. Failed tasks are counted against failed attempts. |
[16/50] [abbrv] hadoop git commit: HADOOP-12548. Read s3a creds from
a Credential Provider. Contributed by Larry McCay.
Posted by ar...@apache.org.
HADOOP-12548. Read s3a creds from a Credential Provider. Contributed by Larry McCay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76fab26c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76fab26c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76fab26c
Branch: refs/heads/HDFS-1312
Commit: 76fab26c5c02cef38924d04136407489fd9457d9
Parents: 3a23dc6
Author: cnauroth <cn...@apache.org>
Authored: Wed Feb 17 10:17:12 2016 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Feb 17 12:19:59 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 212 +++++++++++++------
.../hadoop/fs/s3a/TestS3AConfiguration.java | 151 ++++++++++++-
3 files changed, 301 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76fab26c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index cde5f3b..9269960 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -766,6 +766,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12426. Add Entry point for Kerberos health check
(Steve Loughran via cnauroth)
+ HADOOP-12548. Read s3a creds from a Credential Provider.
+ (Larry McCay via cnauroth)
+
IMPROVEMENTS
HADOOP-12458. Retries is typoed to spell Retires in parts of
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76fab26c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 5ea6bec..b9590ea 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -106,23 +106,11 @@ public class S3AFileSystem extends FileSystem {
workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri,
this.getWorkingDirectory());
- // Try to get our credentials or just connect anonymously
- String accessKey = conf.get(ACCESS_KEY, null);
- String secretKey = conf.get(SECRET_KEY, null);
-
- String userInfo = name.getUserInfo();
- if (userInfo != null) {
- int index = userInfo.indexOf(':');
- if (index != -1) {
- accessKey = userInfo.substring(0, index);
- secretKey = userInfo.substring(index + 1);
- } else {
- accessKey = userInfo;
- }
- }
+ AWSAccessKeys creds = getAWSAccessKeys(name, conf);
AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
- new BasicAWSCredentialsProvider(accessKey, secretKey),
+ new BasicAWSCredentialsProvider(
+ creds.getAccessKey(), creds.getAccessSecret()),
new InstanceProfileCredentialsProvider(),
new AnonymousAWSCredentialsProvider()
);
@@ -146,6 +134,59 @@ public class S3AFileSystem extends FileSystem {
awsConf.setSignerOverride(signerOverride);
}
+ initProxySupport(conf, awsConf, secureConnections);
+
+ initAmazonS3Client(conf, credentials, awsConf);
+
+ maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
+ partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
+ multiPartThreshold = conf.getLong(MIN_MULTIPART_THRESHOLD,
+ DEFAULT_MIN_MULTIPART_THRESHOLD);
+ enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
+
+ if (partSize < 5 * 1024 * 1024) {
+ LOG.error(MULTIPART_SIZE + " must be at least 5 MB");
+ partSize = 5 * 1024 * 1024;
+ }
+
+ if (multiPartThreshold < 5 * 1024 * 1024) {
+ LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB");
+ multiPartThreshold = 5 * 1024 * 1024;
+ }
+
+ int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS);
+ if (maxThreads < 2) {
+ LOG.warn(MAX_THREADS + " must be at least 2: forcing to 2.");
+ maxThreads = 2;
+ }
+ int totalTasks = conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS);
+ if (totalTasks < 1) {
+ LOG.warn(MAX_TOTAL_TASKS + "must be at least 1: forcing to 1.");
+ totalTasks = 1;
+ }
+ long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
+ threadPoolExecutor = new BlockingThreadPoolExecutorService(maxThreads,
+ maxThreads + totalTasks, keepAliveTime, TimeUnit.SECONDS,
+ "s3a-transfer-shared");
+
+ initTransferManager();
+
+ initCannedAcls(conf);
+
+ if (!s3.doesBucketExist(bucket)) {
+ throw new IOException("Bucket " + bucket + " does not exist");
+ }
+
+ initMultipartUploads(conf);
+
+ serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM);
+
+ setConf(conf);
+ }
+
+ void initProxySupport(Configuration conf, ClientConfiguration awsConf,
+ boolean secureConnections) throws IllegalArgumentException,
+ IllegalArgumentException {
String proxyHost = conf.getTrimmed(PROXY_HOST, "");
int proxyPort = conf.getInt(PROXY_PORT, -1);
if (!proxyHost.isEmpty()) {
@@ -185,7 +226,11 @@ public class S3AFileSystem extends FileSystem {
LOG.error(msg);
throw new IllegalArgumentException(msg);
}
+ }
+ private void initAmazonS3Client(Configuration conf,
+ AWSCredentialsProviderChain credentials, ClientConfiguration awsConf)
+ throws IllegalArgumentException {
s3 = new AmazonS3Client(credentials, awsConf);
String endPoint = conf.getTrimmed(ENDPOINT,"");
if (!endPoint.isEmpty()) {
@@ -197,56 +242,27 @@ public class S3AFileSystem extends FileSystem {
throw new IllegalArgumentException(msg, e);
}
}
+ }
- maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
- partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
- multiPartThreshold = conf.getLong(MIN_MULTIPART_THRESHOLD,
- DEFAULT_MIN_MULTIPART_THRESHOLD);
- enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
-
- if (partSize < 5 * 1024 * 1024) {
- LOG.error(MULTIPART_SIZE + " must be at least 5 MB");
- partSize = 5 * 1024 * 1024;
- }
-
- if (multiPartThreshold < 5 * 1024 * 1024) {
- LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB");
- multiPartThreshold = 5 * 1024 * 1024;
- }
-
- int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS);
- if (maxThreads < 2) {
- LOG.warn(MAX_THREADS + " must be at least 2: forcing to 2.");
- maxThreads = 2;
- }
- int totalTasks = conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS);
- if (totalTasks < 1) {
- LOG.warn(MAX_TOTAL_TASKS + "must be at least 1: forcing to 1.");
- totalTasks = 1;
- }
- long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
- threadPoolExecutor = new BlockingThreadPoolExecutorService(maxThreads,
- maxThreads + totalTasks, keepAliveTime, TimeUnit.SECONDS,
- "s3a-transfer-shared");
-
+ private void initTransferManager() {
TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
transferConfiguration.setMinimumUploadPartSize(partSize);
transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
transfers = new TransferManager(s3, threadPoolExecutor);
transfers.setConfiguration(transferConfiguration);
+ }
+ private void initCannedAcls(Configuration conf) {
String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
if (!cannedACLName.isEmpty()) {
cannedACL = CannedAccessControlList.valueOf(cannedACLName);
} else {
cannedACL = null;
}
+ }
- if (!s3.doesBucketExist(bucket)) {
- throw new IOException("Bucket " + bucket + " does not exist");
- }
-
+ private void initMultipartUploads(Configuration conf) {
boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
DEFAULT_PURGE_EXISTING_MULTIPART);
long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE,
@@ -257,10 +273,51 @@ public class S3AFileSystem extends FileSystem {
transfers.abortMultipartUploads(bucket, purgeBefore);
}
+ }
- serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM);
-
- setConf(conf);
+ /**
+ * Return the access key and secret for S3 API use.
+ * Credentials may exist in configuration, within credential providers
+ * or indicated in the UserInfo of the name URI param.
+ * @param name the URI for which we need the access keys.
+ * @param conf the Configuration object to interogate for keys.
+ * @return AWSAccessKeys
+ */
+ AWSAccessKeys getAWSAccessKeys(URI name, Configuration conf)
+ throws IOException {
+ String accessKey = null;
+ String secretKey = null;
+ String userInfo = name.getUserInfo();
+ if (userInfo != null) {
+ int index = userInfo.indexOf(':');
+ if (index != -1) {
+ accessKey = userInfo.substring(0, index);
+ secretKey = userInfo.substring(index + 1);
+ } else {
+ accessKey = userInfo;
+ }
+ }
+ if (accessKey == null) {
+ try {
+ final char[] key = conf.getPassword(ACCESS_KEY);
+ if (key != null) {
+ accessKey = (new String(key)).trim();
+ }
+ } catch(IOException ioe) {
+ throw new IOException("Cannot find AWS access key.", ioe);
+ }
+ }
+ if (secretKey == null) {
+ try {
+ final char[] pass = conf.getPassword(SECRET_KEY);
+ if (pass != null) {
+ secretKey = (new String(pass)).trim();
+ }
+ } catch(IOException ioe) {
+ throw new IOException("Cannot find AWS secret key.", ioe);
+ }
+ }
+ return new AWSAccessKeys(accessKey, secretKey);
}
/**
@@ -341,14 +398,14 @@ public class S3AFileSystem extends FileSystem {
* Create an FSDataOutputStream at the indicated Path with write-progress
* reporting.
* @param f the file name to open
- * @param permission
+ * @param permission the permission to set.
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
- * @throws IOException
+ * @param blockSize the requested block size.
+ * @param progress the progress reporter.
+ * @throws IOException in the event of IO related errors.
* @see #setPermission(Path, FsPermission)
*/
@Override
@@ -377,7 +434,7 @@ public class S3AFileSystem extends FileSystem {
* @param f the existing file to be appended.
* @param bufferSize the size of the buffer to be used.
* @param progress for reporting progress if it is not null.
- * @throws IOException
+ * @throws IOException indicating that append is not supported.
*/
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
@@ -585,7 +642,7 @@ public class S3AFileSystem extends FileSystem {
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
* @return true if delete is successful else false.
- * @throws IOException
+ * @throws IOException due to inability to delete a directory or file.
*/
public boolean delete(Path f, boolean recursive) throws IOException {
if (LOG.isDebugEnabled()) {
@@ -791,7 +848,7 @@ public class S3AFileSystem extends FileSystem {
* Set the current working directory for the given file system. All relative
* paths will be resolved relative to it.
*
- * @param new_dir
+ * @param new_dir the current working directory.
*/
public void setWorkingDirectory(Path new_dir) {
workingDir = new_dir;
@@ -1202,4 +1259,39 @@ public class S3AFileSystem extends FileSystem {
"such as not being able to access the network.");
LOG.info("Error Message: {}" + ace, ace);
}
+
+ /**
+ * This is a simple encapsulation of the
+ * S3 access key and secret.
+ */
+ static class AWSAccessKeys {
+ private String accessKey = null;
+ private String accessSecret = null;
+
+ /**
+ * Constructor.
+ * @param key - AWS access key
+ * @param secret - AWS secret key
+ */
+ public AWSAccessKeys(String key, String secret) {
+ accessKey = key;
+ accessSecret = secret;
+ }
+
+ /**
+ * Return the AWS access key.
+ * @return key
+ */
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ /**
+ * Return the AWS secret key.
+ * @return secret
+ */
+ public String getAccessSecret() {
+ return accessSecret;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76fab26c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
index 25068f8..3db84db 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
@@ -32,7 +32,21 @@ import org.slf4j.LoggerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+import java.io.File;
+import java.net.URI;
+import java.io.IOException;
+
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+
+import org.junit.rules.TemporaryFolder;
+
public class TestS3AConfiguration {
+ private static final String EXAMPLE_ID = "AKASOMEACCESSKEY";
+ private static final String EXAMPLE_KEY =
+ "RGV0cm9pdCBSZ/WQgY2xl/YW5lZCB1cAEXAMPLE";
+
private Configuration conf;
private S3AFileSystem fs;
@@ -44,6 +58,9 @@ public class TestS3AConfiguration {
@Rule
public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+ @Rule
+ public final TemporaryFolder tempDir = new TemporaryFolder();
+
/**
* Test if custom endpoint is picked up.
* <p/>
@@ -59,7 +76,7 @@ public class TestS3AConfiguration {
* @throws Exception
*/
@Test
- public void TestEndpoint() throws Exception {
+ public void testEndpoint() throws Exception {
conf = new Configuration();
String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
if (endpoint.isEmpty()) {
@@ -85,7 +102,7 @@ public class TestS3AConfiguration {
}
@Test
- public void TestProxyConnection() throws Exception {
+ public void testProxyConnection() throws Exception {
conf = new Configuration();
conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
conf.set(Constants.PROXY_HOST, "127.0.0.1");
@@ -103,7 +120,7 @@ public class TestS3AConfiguration {
}
@Test
- public void TestProxyPortWithoutHost() throws Exception {
+ public void testProxyPortWithoutHost() throws Exception {
conf = new Configuration();
conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
conf.setInt(Constants.PROXY_PORT, 1);
@@ -120,7 +137,7 @@ public class TestS3AConfiguration {
}
@Test
- public void TestAutomaticProxyPortSelection() throws Exception {
+ public void testAutomaticProxyPortSelection() throws Exception {
conf = new Configuration();
conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
conf.set(Constants.PROXY_HOST, "127.0.0.1");
@@ -145,7 +162,7 @@ public class TestS3AConfiguration {
}
@Test
- public void TestUsernameInconsistentWithPassword() throws Exception {
+ public void testUsernameInconsistentWithPassword() throws Exception {
conf = new Configuration();
conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
conf.set(Constants.PROXY_HOST, "127.0.0.1");
@@ -177,4 +194,128 @@ public class TestS3AConfiguration {
}
}
}
+
+ @Test
+ public void testCredsFromCredentialProvider() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ S3AFileSystem s3afs = new S3AFileSystem();
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ S3AFileSystem.AWSAccessKeys creds =
+ s3afs.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getAccessKey());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
+ }
+
+ void provisionAccessKeys(final Configuration conf) throws Exception {
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.ACCESS_KEY,
+ EXAMPLE_ID.toCharArray());
+ provider.createCredentialEntry(Constants.SECRET_KEY,
+ EXAMPLE_KEY.toCharArray());
+ provider.flush();
+ }
+
+ @Test
+ public void testCredsFromUserInfo() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ S3AFileSystem s3afs = new S3AFileSystem();
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123:456@foobar");
+ S3AFileSystem.AWSAccessKeys creds =
+ s3afs.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getAccessKey());
+ assertEquals("SecretKey incorrect.", "456", creds.getAccessSecret());
+ }
+
+ @Test
+ public void testIDFromUserInfoSecretFromCredentialProvider()
+ throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ S3AFileSystem s3afs = new S3AFileSystem();
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123@foobar");
+ S3AFileSystem.AWSAccessKeys creds =
+ s3afs.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getAccessKey());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
+ }
+
+ @Test
+ public void testSecretFromCredentialProviderIDFromConfig() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.SECRET_KEY,
+ EXAMPLE_KEY.toCharArray());
+ provider.flush();
+
+ S3AFileSystem s3afs = new S3AFileSystem();
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID);
+ S3AFileSystem.AWSAccessKeys creds =
+ s3afs.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getAccessKey());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
+ }
+
+ @Test
+ public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.ACCESS_KEY,
+ EXAMPLE_ID.toCharArray());
+ provider.flush();
+
+ S3AFileSystem s3afs = new S3AFileSystem();
+ conf.set(Constants.SECRET_KEY, EXAMPLE_KEY);
+ S3AFileSystem.AWSAccessKeys creds =
+ s3afs.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getAccessKey());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getAccessSecret());
+ }
}
[02/50] [abbrv] hadoop git commit: YARN-4653. Document YARN security
model from the perspective of Application Developers. Contributed by Steve
Loughran
Posted by ar...@apache.org.
YARN-4653. Document YARN security model from the perspective of Application Developers. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dea90c9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dea90c9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dea90c9a
Branch: refs/heads/HDFS-1312
Commit: dea90c9a86d0b17f36d0bdf24ca0c789dd1de2b6
Parents: ec12ce8
Author: Jian He <ji...@apache.org>
Authored: Sun Feb 14 17:13:15 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Sun Feb 14 17:13:15 2016 +0800
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../site/markdown/YarnApplicationSecurity.md | 560 +++++++++++++++++++
3 files changed, 564 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dea90c9a/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index b7db5a3..00c5e65 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -126,6 +126,7 @@
<item name="Web Application Proxy" href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
<item name="Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html"/>
<item name="Writing YARN Applications" href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
+ <item name="YARN Application Security" href="hadoop-yarn/hadoop-yarn-site/YarnApplicationSecurity.html"/>
<item name="NodeManager" href="hadoop-yarn/hadoop-yarn-site/NodeManager.html"/>
<item name="DockerContainerExecutor" href="hadoop-yarn/hadoop-yarn-site/DockerContainerExecutor.html"/>
<item name="Using CGroups" href="hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dea90c9a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1489976..1547f10 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1433,6 +1433,9 @@ Release 2.7.3 - UNRELEASED
YARN-4492. Add documentation for preemption supported in Capacity
scheduler (Naganarasimha G R via jlowe)
+ YARN-4653. Document YARN security model from the perspective of
+ Application Developers. (Steve Loughran via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dea90c9a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md
new file mode 100644
index 0000000..640139f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md
@@ -0,0 +1,560 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# YARN Application Security
+
+Anyone writing a YARN application needs to understand the process, in order
+to write short-lived applications or long-lived services. They also need to
+start testing on secure clusters during early development stages, in order
+to write code that actually works.
+
+## How YARN Security works
+
+YARN Resource Managers (RMs) and Node Managers (NMs) co-operate to execute
+the user's application with the identity and hence access rights of that user.
+
+The (active) Resource Manager:
+
+1. Finds space in a cluster to deploy the core of the application,
+the Application Master (AM).
+
+1. Requests that the NM on that node allocate a container and start the AM in it.
+
+1. Communicates with the AM, so that the AM can request new containers and
+manipulate/release current ones, and to provide notifications about allocated
+and running containers.
+
+The Node Managers:
+
+1. *Localize* resources: Download from HDFS or other filesystem into a local directory. This
+is done using the delegation tokens attached to the container launch context. (For non-HDFS
+resources, using other credentials such as object store login details in cluster configuration
+files)
+
+1. Start the application as the user.
+
+1. Monitor the application and report failure to the RM.
+
+To execute code in the cluster, a YARN application must:
+
+1. Have a client-side application which sets up the `ApplicationSubmissionContext`
+detailing what is to be launched. This includes:
+
+ * A list of files in the cluster's filesystem to be "localized".
+ * The environment variables to set in the container.
+ * The commands to execute in the container to start the application.
+ * Any security credentials needed by YARN to launch the application.
+ * Any security credentials needed by the application to interact
+ with any Hadoop cluster services and applications.
+
+1. Have an Application Master which, when launched, registers with
+the YARN RM and listens for events. Any AM which wishes to execute work in
+other containers must request them off the RM, and, when allocated, create
+a `ContainerLaunchContext` containing the command to execute, the
+environment to execute the command, binaries to localize and all relevant
+security credentials.
+
+1. Even with the NM handling the localization process, the AM must itself
+be able to retrieve the security credentials supplied at launch time so
+that it itself may work with HDFS and any other services, and to pass some or
+all of these credentials down to the launched containers.
+
+### Acquiring and Adding tokens to a YARN Application
+
+The delegation tokens which a YARN application needs must be acquired
+from a program executing as an authenticated user. For a YARN application,
+this means the user launching the application. It is the client-side part
+of the YARN application which must do this:
+
+1. Log in via `UserGroupInformation`.
+1. Identify all tokens which must be acquired.
+1. Request these tokens from the specific Hadoop services.
+1. Marshall all tokens into a byte buffer.
+1. Add them to the `ContainerLaunchContext` within the `ApplicationSubmissionContext`.
+
+Which tokens are required? Normally, at least a token to access HDFS.
+
+An application must request a delegation token from every filesystem with
+which it intends to interact —including the cluster's main FS.
+`FileSystem.addDelegationTokens(renewer, credentials)` can be used to collect these;
+it is a no-op on those filesystems which do not issue tokens (including
+non-kerberized HDFS clusters).
+
+Applications talking to other services, such as Apache HBase and Apache Hive,
+must request tokens from these services, using the libraries of these
+services to acquire delegation tokens. All tokens can be added to the same
+set of credentials, then saved to a byte buffer for submission.
+
+The Application Timeline Server also needs a delegation token. This is handled
+automatically on AM launch.
+
+### Extracting tokens within the AM
+
+When the Application Master is launched and any of the UGI/Hadoop operations
+which trigger a user login invoked, the UGI class will automatically load in all tokens
+saved in the file named by the environment variable `HADOOP_TOKEN_FILE_LOCATION`.
+
+This happens on an insecure cluster along with a secure one, and on a secure
+cluster even if a keytab is used by the application. Why? Because the
+AM/RM token needed to authenticate the application with the YARN RM is always
+supplied this way.
+
+This means you have a relative similar workflow across secure and insecure clusters.
+
+1. Suring AM startup, log in to Kerberos.
+A call to `UserGroupInformation.isSecurityEnabled()` will trigger this operation.
+
+1. Enumerate the current user's credentials, through a call of
+`UserGroupInformation.getCurrentUser().getCredentials()`.
+
+1. Filter out the AMRM token, resulting in a new set of credentials. In an
+insecure cluster, the list of credentials will now be empty; in a secure cluster
+they will contain
+
+1. Set the credentials of all containers to be launched to this (possibly empty)
+list of credentials.
+
+1. If the filtered list of tokens to renew, is non-empty start up a thread
+to renew them.
+
+### Token Renewal
+
+Tokens *expire*: they have a limited lifespan. An application wishing to
+use a token past this expiry date must *renew* the token before the token
+expires.
+
+Hadoop automatically sets up a delegation token renewal thread when needed,
+the `DelegationTokenRenewer`.
+
+It is the responsibility of the application to renew all tokens other
+than the AMRM and timeline tokens.
+
+Here are the different strategies
+
+1. Don't. Rely on the lifespan of the application being so short that token
+renewal is not needed. For applications whose life can always be measured
+in minutes or tens of minutes, this is a viable strategy.
+
+1. Start a background thread/Executor to renew the tokens at a regular interval.
+This what most YARN applications do.
+
+## Other Aspects of YARN Security
+
+
+### AM/RM Token Refresh
+
+The AM/RM token is renewed automatically; the AM pushes out a new token
+to the AM within an `allocate` message. Consult the `AMRMClientImpl` class
+to see the process. *Your AM code does not need to worry about this process*
+
+### Token Renewal on AM Restart
+
+Even if an application is renewing tokens regularly, if an AM fails and is
+restarted, it gets restarted from that original
+`ApplicationSubmissionContext`. The tokens there may have expired, so localization
+may fail, even before the issue of credentials to talk to other services.
+
+How is this problem addressed? The YARN Resource Manager gets a new token
+for the node managers, if needed.
+
+More precisely
+
+1. The token passed by the RM to the NM for localization is refreshed/updated as needed.
+1. Tokens in the app launch context for use by the application are *not* refreshed.
+That is, if it has an out of date HDFS token —that token is not renewed. This
+also holds for tokens for for Hive, HBase, etc.
+1. Therefore, to survive AM restart after token expiry, your AM has to get the
+NMs to localize the keytab or make no HDFS accesses until (somehow) a new token has been passed to them from a client.
+
+This is primarily an issue for long-lived services (see below).
+
+### Unmanaged Application Masters
+
+Unmanaged application masters are not launched in a container set up by
+the RM and NM, so cannot automatically pick up an AM/RM token at launch time.
+The `YarnClient.getAMRMToken()` API permits an Unmanaged AM to request an AM/RM
+token. Consult `UnmanagedAMLauncher` for the specifics.
+
+### Identity on an insecure cluster: `HADOOP_USER_NAME`
+
+In an insecure cluster, the application will run as the identity of
+the account of the node manager, typically something such as `yarn`
+or `mapred`. By default, the application will access HDFS
+as that user, with a different home directory, and with
+a different user identified in audit logs and on file system owner attributes.
+
+This can be avoided by having the client identify the identify of the
+HDFS/Hadoop user under which the application is expected to run. *This
+does not affect the OS-level user or the application's access rights
+to the local machine*.
+
+When Kerberos is disabled, the identity of a user is picked up
+by Hadoop first from the environment variable `HADOOP_USER_NAME`,
+then from the OS-level username (e.g. the system property `user.name`).
+
+YARN applications should propagate the user name of the user launching
+an application by setting this environment variable.
+
+```java
+Map<String, String> env = new HashMap<>();
+String userName = UserGroupInformation.getCurrentUser().getUserName();
+env.put(UserGroupInformation.HADOOP_USER_NAME, userName);
+containerLaunchContext.setEnvironment(env);
+```
+
+Note that this environment variable is picked up in all applications
+which talk to HDFS via the hadoop libraries. That is, if set, it
+is the identity picked up by HBase and other applications executed
+within the environment of a YARN container within which this environment
+variable is set.
+
+### Oozie integration and `HADOOP_TOKEN_FILE_LOCATION`
+
+Apache Oozie can launch an application in a secure cluster either by acquiring
+all relevant credentials, saving them to a file in the local filesystem,
+then setting the path to this file in the environment variable
+`HADOOP_TOKEN_FILE_LOCATION`. This is of course the same environment variable
+passed down by YARN in launched containers, as is similar content: a byte
+array with credentials.
+
+Here, however, the environment variable is set in the environment
+executing the YARN client. This client must use the token information saved
+in the named file *instead of acquiring any tokens of its own*.
+
+Loading in the token file is automatic: UGI does it during user login.
+
+The client is then responsible for passing the same credentials into the
+AM launch context. This can be done simply by passing down the current
+credentials.
+
+```java
+credentials = new Credentials(
+ UserGroupInformation.getCurrentUser().getCredentials());
+```
+
+### Timeline Server integration
+
+The [Application Timeline Server](TimelineServer.html) can be deployed as a secure service
+—in which case the application will need the relevant token to authenticate with
+it. This process is handled automatically in `YarnClientImpl` if ATS is
+enabled in a secure cluster. Similarly, the AM-side `TimelineClient` YARN service
+class manages token renewal automatically via the ATS's SPNEGO-authenticated REST API.
+
+If you need to prepare a set of delegation tokens for a YARN application launch
+via Oozie, this can be done via the timeline client API.
+
+```java
+try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) {
+ timelineClient.init(conf);
+ timelineClient.start();
+ Token<TimelineDelegationTokenIdentifier> token =
+ timelineClient.getDelegationToken(rmprincipal));
+ credentials.addToken(token.getService(), token);
+}
+```
+
+### Cancelling Tokens
+
+Applications *may* wish to cancel tokens they hold when terminating their AM.
+This ensures that the tokens are no-longer valid.
+
+This is not mandatory, and as a clean shutdown of a YARN application cannot
+be guaranteed, it is not possible to guarantee that the tokens will always
+be during application termination. However, it does reduce the window of
+vulnerability to stolen tokens.
+
+## Securing Long-lived YARN Services
+
+There is a time limit on all token renewals, after which tokens won't renew,
+causing the application to stop working. This is somewhere between seventy-two
+hours and seven days.
+
+Any YARN service intended to run for an extended period of time *must* have
+a strategy for renewing credentials.
+
+Here are the strategies:
+
+### Pre-installed Keytabs for AM and containers
+
+A keytab is provided for the application's use on every node.
+
+This is done by:
+
+1. Installing it in every cluster node's local filesystem.
+1. Providing the path to this in a configuration option.
+1. The application loading the credentials via
+ `UserGroupInformation.loginUserFromKeytab()`.
+
+The keytab must be in a secure directory path, where
+only the service (and other trusted accounts) can read it. Distribution
+becomes a responsibility of the cluster operations team.
+
+This is effectively how all static Hadoop applications get their security credentials.
+
+### Keytabs for AM and containers distributed via YARN
+
+
+1. A keytab is uploaded to HDFS.
+
+1. When launching the AM, the keytab is listed as a resource to localize to
+the AM's container.
+
+1. The Application Master is configured with the relative path to the keytab,
+and logs in with `UserGroupInformation.loginUserFromKeytab()`.
+
+1. When the AM launches the container, it lists the HDFS path to the keytab
+as a resource to localize.
+
+1. It adds the HDFS delegation token to the container launch context, so
+that the keytab and other application files can be localized.
+
+1. Launched containers must themselves log in via
+ `UserGroupInformation.loginUserFromKeytab()`. UGI handles the login, and
+ schedules a background thread to relogin the user periodically.
+
+1. Token creation is handled automatically in the Hadoop IPC and REST APIs,
+the containers stay logged in via kerberos for their entire duration.
+
+This avoids the administration task of installing keytabs for specific services
+across the entire cluster.
+
+It does require the client to have access to the keytab
+and, as it is uploaded to the distributed filesystem, must be secured through
+the appropriate path permissions/ACLs.
+
+As all containers have access to the keytab, all code executing in the containers
+has to be trusted. Malicious code (or code escaping some form of sandbox)
+could read the keytab, and hence have access to the cluster until the keys
+expire or are revoked.
+
+This is the strategy implemented by Apache Slider (incubating).
+
+### AM keytab distributed via YARN; AM regenerates delegation tokens for containers.
+
+1. A keytab is uploaded to HDFS by the client.
+
+1. When launching the AM, the keytab is listed as a resource to localize to
+the AM's container.
+
+1. The Application Master is configured with the relative path to the keytab,
+and logs in with `UserGroupInformation.loginUserFromKeytab()`. The UGI
+codepath will still automatically load the file references by
+`$HADOOP_TOKEN_FILE_LOCATION`, which is how the AMRM token is picked up.
+
+1. When the AM launches a container, it acquires all the delegation tokens
+needed by that container, and adds them to the container's container launch context.
+
+1. Launched containers must load the delegation tokens from `$HADOOP_TOKEN_FILE_LOCATION`,
+and use them (including renewals) until they can no longer be renewed.
+
+1. The AM must implement an IPC interface which permits containers to request
+a new set of delegation tokens; this interface must itself use authentication
+and ideally wire encryption.
+
+1. Before a delegation token is due to expire, the processes running in the containers
+must request new tokens from the Application Master over the IPC channel.
+
+1. When the containers need the new tokens, the AM, logged in with a keytab,
+ asks the various cluster services for new tokens.
+
+(Note there is an alternative direction for refresh operations: from AM
+ to the containers, again over whatever IPC channel is implemented between
+ AM and containers). The rest of the algorithm: AM regenerated tokens passed
+ to containers over IPC.
+
+This is the strategy used by Apache Spark 1.5+, with a netty-based protocol
+between containers and the AM for token updates.
+
+Because only the AM has direct access to the keytab, it is less exposed.
+Code running in the containers only has access to the delegation tokens.
+
+However, those containers will have access to HDFS from the tokens
+passed in at container launch, so will have access to the copy of the keytab
+used for launching the AM. While the AM could delete that keytab on launch,
+doing so would stop YARN being able to successfully relaunch the AM after any
+failure.
+
+### Client-side Token Push
+
+This strategy may be the sole one acceptable to a strict operations team: a client process
+running on an account holding a Kerberos TGT negotiates with all needed cluster services
+for new delegation tokens, tokens which are then pushed out to the Application Master via
+some RPC interface.
+
+This does require the client process to be re-executed on a regular basis; a cron or Oozie job
+can do this. The AM will need to implement an IPC API over which renewed
+tokens can be provided. (Note that as Oozie can collect the tokens itself,
+all the updater application needs to do whenever executed is set up an IPC
+connection with the AM and pass up the current user's credentials).
+
+## Securing YARN Application Web UIs and REST APIs
+
+YARN provides a straightforward way of giving every YARN application SPNEGO authenticated
+web pages: it implements SPNEGO authentication in the Resource Manager Proxy.
+
+YARN web UI are expected to load the AM proxy filter when setting up its web UI; this filter
+will redirect all HTTP Requests coming from any host other than the RM Proxy hosts to an
+RM proxy, to which the client app/browser must re-issue the request. The client will authenticate
+against the principal of the RM Proxy (usually `yarn`), and, once authenticated, have its
+request forwared.
+
+As a result, all client interactions are SPNEGO-authenticated, without the YARN application
+itself needing any kerberos principal for the clients to authenticate against.
+
+Known weaknesses in this approach are:
+
+1. As calls coming from the proxy hosts are not redirected, any application running
+on those hosts has unrestricted access to the YARN applications. This is why in a secure cluster
+the proxy hosts *must* run on cluster nodes which do not run end user code (i.e. not run YARN
+NodeManagers and hence schedule YARN containers, nor support logins by end users).
+
+1. The HTTP requests between proxy and YARN RM Server are not currently encrypted.
+That is: HTTPS is not supported.
+
+## Securing YARN Application REST APIs
+
+YARN REST APIs running on the same port as the registered web UI of a YARN application are
+automatically authenticated via SPNEGO authentication in the RM proxy.
+
+Any REST endpoint (and equally, any web UI) brought up on a different port does not
+support SPNEGO authentication unless implemented in the YARN application itself.
+
+## Checklist for YARN Applications
+
+Here is the checklist of core actions which a YARN application must do
+to successfully launch in a YARN cluster.
+
+### Client
+
+`[ ]` Client checks for security being enabled via `UserGroupInformation.isSecurityEnabled()`
+
+In a secure cluster:
+
+`[ ]` If `HADOOP_TOKEN_FILE_LOCATION` is unset, client acquires delegation tokens
+ for the local filesystems, with the RM principal set as the renewer.
+
+`[ ]` If `HADOOP_TOKEN_FILE_LOCATION` is unset, client acquires delegation tokens
+for all other services to be used in the YARN application.
+
+`[ ]` If `HADOOP_TOKEN_FILE_LOCATION` is set, client uses the current user's credentials
+as the source of all tokens to be added to the container launch context.
+
+`[ ]` Client sets all tokens on AM `ContainerLaunchContext.setTokens()`.
+
+`[ ]` Recommended: if it is set in the client's environment,
+client sets the environment variable `HADOOP_JAAS_DEBUG=true`
+in the Container launch context of the AM.
+
+In an insecure cluster:
+
+`[ ]` Propagate local username to YARN AM, hence HDFS identity via the
+`HADOOP_USER_NAME` environment variable.
+
+### App Master
+
+`[ ]` In a secure cluster, AM retrieves security tokens from `HADOOP_TOKEN_FILE_LOCATION`
+environment variable (automatically done by UGI).
+
+`[ ]` A copy the token set is filtered to remove the AM/RM token and any timeline
+token.
+
+`[ ]` A thread or executor is started to renew threads on a regular basis.
+
+`[ ]` Recommended: AM cancels tokens when application completes.
+
+### Container Launch by AM
+
+`[ ]` Tokens to be passed to containers are passed via
+`ContainerLaunchContext.setTokens()`.
+
+`[ ]` In an insecure cluster, propagate the `HADOOP_USER_NAME` environment variable.
+
+`[ ]` Recommended: AM sets the environment variable `HADOOP_JAAS_DEBUG=true`
+in the Container launch context if it is set in the AM's environment.
+
+### Launched Containers
+
+`[ ]` Call `UserGroupInformation.isSecurityEnabled()` to trigger security setup.
+
+`[ ]` A thread or executor is started to renew threads on a regular basis.
+
+### YARN service
+
+`[ ]` Application developers have chosen and implemented a token renewal strategy:
+shared keytab, AM keytab or client-side token refresh.
+
+`[ ]` In a secure cluster, the keytab is either already in HDFS (and checked for),
+or it is in the local FS of the client, in which case it must be uploaded and added to
+the list of resources to localize.
+
+`[ ]` If stored in HDFS, keytab permissions should be checked. If the keytab
+is readable by principals other than the current user, warn,
+and consider actually failing the launch (similar to the normal `ssh` application.)
+
+`[ ]` Client acquires HDFS delegation token and and attaches to the AM Container
+Launch Context,
+
+`[ ]` AM logs in as principal in keytab via `loginUserFromKeytab()`.
+
+`[ ]` (AM extracts AM/RM token from the `HADOOP_TOKEN_FILE_LOCATION` environment
+variable).
+
+`[ ]` For launched containers, either the keytab is propagated, or
+the AM acquires/attaches all required delegation tokens to the Container Launch
+context alongside the HDFS delegation token needed by the NMs.
+
+## Testing YARN applications in a secure cluster.
+
+It is straightforward to be confident that a YARN application works in secure
+cluster. The process to do so is: test on a secure cluster.
+
+Even a single VM-cluster can be set up with security enabled. If doing so,
+we recommend turning security up to its strictest, with SPNEGO-authenticated
+Web UIs (and hence RM Proxy), as well as IPC wire encryption. Setting the
+kerberos token expiry to under an hour will find kerberos expiry problems
+early —so is also recommended.
+
+`[ ]` Application launched in secure cluster.
+
+`[ ]` Launched application runs as user submitting job (tip: log `user.name`
+system property in AM).
+
+`[ ]` Web browser interaction verified in secure cluster.
+
+`[ ]` REST client interation (GET operations) tested.
+
+`[ ]` Application continues to run after Kerberos Token expiry.
+
+`[ ]` Application does not launch if user lacks Kerberos credentials.
+
+`[ ]` If the application supports the timeline server, verify that it publishes
+events in a secure cluster.
+
+`[ ]` If the application integrates with other applications, such as HBase or Hive,
+verify that the interaction works in a secure cluster.
+
+`[ ]` If the application communicates with remote HDFS clusters, verify
+that it can do so in a secure cluster (i.e. that the client extracted any
+delegation tokens for this at launch time)
+
+## Important
+
+*If you don't test your YARN application in a secure Hadoop cluster,
+it won't work.*
+
+And without those tests: *your users will be the ones to find out
+that your application doesn't work in a secure cluster.*
+
+Bear that in mind when considering how much development effort to put into
+Kerberos support.
[21/50] [abbrv] hadoop git commit: YARN-4690. Skip object allocation
in FSAppAttempt#getResourceUsage when possible (Ming Ma via sjlee)
Posted by ar...@apache.org.
YARN-4690. Skip object allocation in FSAppAttempt#getResourceUsage when possible (Ming Ma via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7de70680
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7de70680
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7de70680
Branch: refs/heads/HDFS-1312
Commit: 7de70680fe44967e2afc92ba4c92f8e7afa7b151
Parents: a0c95b5
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Feb 17 20:55:21 2016 -0800
Committer: Sangjin Lee <sj...@twitter.com>
Committed: Wed Feb 17 20:55:21 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/scheduler/fair/FSAppAttempt.java | 7 ++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7de70680/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c172054..fa0ba44 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -2362,6 +2362,9 @@ Release 2.6.5 - UNRELEASED
OPTIMIZATIONS
+ YARN-4690. Skip object allocation in FSAppAttempt#getResourceUsage when
+ possible (Ming Ma via sjlee)
+
BUG FIXES
Release 2.6.4 - 2016-02-11
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7de70680/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 488f34e..7e0a693 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -890,7 +890,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public Resource getResourceUsage() {
// Here the getPreemptedResources() always return zero, except in
// a preemption round
- return Resources.subtract(getCurrentConsumption(), getPreemptedResources());
+ // In the common case where preempted resource is zero, return the
+ // current consumption Resource object directly without calling
+ // Resources.subtract which creates a new Resource object for each call.
+ return getPreemptedResources().equals(Resources.none()) ?
+ getCurrentConsumption() :
+ Resources.subtract(getCurrentConsumption(), getPreemptedResources());
}
@Override
[39/50] [abbrv] hadoop git commit: MAPREDUCE-6637. Testcase Failure :
TestFileInputFormat.testSplitLocationInfo. Contributed by Brahma Reddy
Battula.
Posted by ar...@apache.org.
MAPREDUCE-6637. Testcase Failure : TestFileInputFormat.testSplitLocationInfo. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37577852
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37577852
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37577852
Branch: refs/heads/HDFS-1312
Commit: 37577852ba54ba506b0b6c4db5e95d176f5e9b63
Parents: 18f9b77
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Feb 19 16:33:17 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Feb 19 16:33:17 2016 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/mapred/TestFileInputFormat.java | 2 +-
.../apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37577852/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a6031c0..bf9d824 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -729,6 +729,9 @@ Release 2.7.3 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-6637. Testcase Failure : TestFileInputFormat.testSplitLocationInfo.
+ (Brahma Reddy Battula via wang)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37577852/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
index ba636b6..141086d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
@@ -243,7 +243,7 @@ public class TestFileInputFormat {
}
@Override
- public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
+ public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len)
throws IOException {
return new BlockLocation[] {
new BlockLocation(new String[] { "localhost:50010", "otherhost:50010" },
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37577852/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
index 3f877f1..dff0e0d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
@@ -422,7 +422,7 @@ public class TestFileInputFormat {
}
@Override
- public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
+ public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len)
throws IOException {
return new BlockLocation[] {
new BlockLocation(new String[] { "localhost:50010", "otherhost:50010" },
[26/50] [abbrv] hadoop git commit: HADOOP-12817. Enable TLS v1.1 and
1.2 (rkanter)
Posted by ar...@apache.org.
HADOOP-12817. Enable TLS v1.1 and 1.2 (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a365a394
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a365a394
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a365a394
Branch: refs/heads/HDFS-1312
Commit: a365a3941cf96a31c289cb22678a602738880f74
Parents: 5d1889a
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Feb 18 11:09:50 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Feb 18 11:09:50 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java | 3 ++-
.../hadoop-common/src/main/resources/core-default.xml | 4 ++--
.../src/site/markdown/EncryptedShuffle.md | 2 +-
4 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a365a394/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9269960..41ba87d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -691,6 +691,8 @@ Release 2.9.0 - UNRELEASED
HADOOP-12710. Remove dependency on commons-httpclient for TestHttpServerLogs
(Wei-Chiu Chuang via iwasakims)
+ HADOOP-12817. Enable TLS v1.1 and 1.2 (rkanter)
+
BUG FIXES
HADOOP-12605. Fix intermittent failure of TestIPC.testIpcWithReaderQueuing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a365a394/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index edec347..518de80 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -69,7 +69,8 @@ public class SSLFactory implements ConnectionConfigurator {
public static final String SSL_ENABLED_PROTOCOLS =
"hadoop.ssl.enabled.protocols";
- public static final String DEFAULT_SSL_ENABLED_PROTOCOLS = "TLSv1";
+ public static final String DEFAULT_SSL_ENABLED_PROTOCOLS =
+ "TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2";
private Configuration conf;
private Mode mode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a365a394/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 34e1236..dd4919c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1644,9 +1644,9 @@ for ldap providers in the same way as above does.
<property>
<name>hadoop.ssl.enabled.protocols</name>
- <value>TLSv1</value>
+ <value>TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2</value>
<description>
- Protocols supported by the ssl.
+ The supported SSL protocols.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a365a394/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
index fddd84f..f679781 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
@@ -44,7 +44,7 @@ To enable encrypted shuffle, set the following properties in core-site.xml of al
| `hadoop.ssl.keystores.factory.class` | `org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory` | The KeyStoresFactory implementation to use |
| `hadoop.ssl.server.conf` | `ssl-server.xml` | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
| `hadoop.ssl.client.conf` | `ssl-client.xml` | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
-| `hadoop.ssl.enabled.protocols` | `TLSv1` | The supported SSL protocols (JDK6 can use **TLSv1**, JDK7+ can use **TLSv1,TLSv1.1,TLSv1.2**) |
+| `hadoop.ssl.enabled.protocols` | `TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2` | The supported SSL protocols |
**IMPORTANT:** Currently requiring client certificates should be set to false. Refer the [Client Certificates](#Client_Certificates) section for details.
[30/50] [abbrv] hadoop git commit: HADOOP-11031. Design Document for
Credential Provider API. Contributed by Larry McCay.
Posted by ar...@apache.org.
HADOOP-11031. Design Document for Credential Provider API. Contributed by Larry McCay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ab76580
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ab76580
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ab76580
Branch: refs/heads/HDFS-1312
Commit: 8ab7658025616d154decd97578de3327ec6f9109
Parents: c7ac25e
Author: cnauroth <cn...@apache.org>
Authored: Thu Feb 18 14:05:18 2016 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Thu Feb 18 14:06:38 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../src/site/markdown/CredentialProviderAPI.md | 132 +++++++++++++++++++
hadoop-project/src/site/site.xml | 1 +
.../src/site/markdown/tools/hadoop-aws/index.md | 45 +++++++
4 files changed, 181 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ab76580/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6cdcb7a..91ef68e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1131,6 +1131,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12776. Remove getaclstatus call for non-acl commands in getfacl.
(Brahma Reddy Battula via vinayakumarb)
+ HADOOP-11031. Design Document for Credential Provider API.
+ (Larry McCay via cnauroth)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ab76580/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
new file mode 100644
index 0000000..eccc205
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -0,0 +1,132 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+CredentialProvider API Guide
+=====================
+
+---
+
+ - [Overview](#Overview)
+ - [Usage](#Usage)
+ - [Usage Overview](#Usage_Overview)
+ - [Supported Features](#Supported_Features)
+ - [Credential Management](#Credential_Management)
+ - [The hadoop credential Command](#The_hadoop_credential_Command)
+
+---
+
+Overview
+--------
+
+ The CredentialProvider API is an SPI framework for plugging in extensible
+ credential providers. Credential providers are used to separate the use of
+ sensitive tokens, secrets and passwords from the details of their storage and
+ management. The ability to choose various storage mechanisms for protecting
+ these credentials allows us to keep such sensitive assets out of clear text,
+ away from prying eyes and potentially to be managed by third party solutions.
+
+ This document aims to describe the design of the CredentialProvider API, the
+ out of the box implementations, where they are used and how to adopt their use.
+
+Usage
+-----
+### Usage Overview
+Let's provide a quick overview of the use of the credential provider framework for protecting passwords or other sensitive tokens in hadoop.
+
+##### Why is it used?
+There are certain deployments that are very sensitive to how sensitive tokens like passwords are stored and managed within the cluster. For instance, there may be security best practices and policies in place that require such things to never be stored in clear text, for example. Enterprise deployments may be required to use a preferred solution for managing credentials and we need a way to plug in integrations for them.
+
+##### General Usage Pattern
+There are numerous places within the Hadoop project and ecosystem that can leverage the credential provider API today and the number continues to grow. In general, the usage pattern consists of the same requirements and flow.
+
+1. Provision credentials within provider specific stores. This provisioning may be accomplished through the hadoop credential command or possibly through provider specific management tools.
+2. Configure the credential provider path property. The provider path property `hadoop.security.credential.provider.path` is a comma separated list of one or more credential provider URIs that is traversed while trying to resolve a credential alias.
+ - This property may be configured within core-site.xml or a component specific configuration file that is merged with core-site.xml.
+ - For command line interfaces, such as that for DistCp, the property can be added with a hadoop system property ("-D *property=value*") and dynamically added to the Configuration.
+3. Features or components that leverage the new [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method to resolve their credentials will automatically pick up support for the credential provider API.
+ - By using the same property names as are used for existing clear text passwords, this mechanism allows for the migration to credential providers while providing backward compatibility for clear text.
+ - The entire credential provider path is interrogated before falling back to clear text passwords in config.
+4. Features or components that do not use the hadoop Configuration class for config or have other internal uses for the credential providers may choose to write to the CredentialProvider API itself. An example of its use will be included in this document but may also be found within [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) and within the unit tests of features that have added support and need to provision credentials for the tests.
+
+##### Provision Credentials
+Example: ssl.server.keystore.password
+
+```
+ hadoop credential create ssl.server.keystore.password -value 123
+ -provider localjceks://file/home/lmccay/aws.jceks
+```
+
+Note that the alias names are the same as the configuration properties that were used to get the
+credentials from the Configuration.get method. Reusing these names allows for intuitive
+migration to the use of credential providers and fall back logic for backward compatibility.
+
+##### Configuring the Provider Path
+Now, we need to make sure that this provisioned credential store is known at runtime by the
+[Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method. If there is no credential provider path configuration then
+getPassword will skip the credential provider API interrogation. So, it is important that the
+following be configured within core-site.xml or your component's equivalent.
+
+```
+ <property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value>localjceks://file/home/lmccay/aws.jceks</value>
+ <description>Path to interrogate for protected credentials.</description>
+ </property>
+```
+
+A couple additional things to note about the provider path:
+
+1. The scheme is used to indicate the type of provider in the above case the
+ localjceks provider does not have a dependency on the Hadoop fs abstraction
+ and is needed sometimes to avoid a recursive dependency. Another provider
+ represented by jceks, does use the Hadoop fs abstraction and therefore has
+ support for keystores provisioned within HDFS. A third provider type is the
+ user type. This provider can manage credentials stored within the Credentials
+ file for a process.
+2. The path configuration accepts a comma separated path of providers or
+ credential stores. The [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method will walk through
+ all of the providers until it resolves the alias or exhausts the list.
+ Depending on the runtime needs for credentials, we may need to configure
+ a chain of providers to check.
+
+In summary, first, provision the credentials into a provider then configure the provider for use by a feature or component and it will often just be picked up through the use of the [Configuration.getPassword](../../api/org/apache/hadoop/conf/Configuration.html#getPassword-java.lang.String-) method.
+
+##### Supported Features
+| Feature\Component | Description | Link |
+|:---- |:---- |:---|
+|LDAPGroupsMapping |LDAPGroupsMapping is used to look up the groups for a given user in LDAP. The CredentialProvider API is used to protect the LDAP bind password and those needed for SSL.|TODO|
+|SSL Passwords |FileBasedKeyStoresFactory leverages the credential provider API in order to resolve the SSL related passwords.|TODO|
+|HDFS |DFSUtil leverages Configuration.getPassword method to use the credential provider API and/or fallback to the clear text value stored in ssl-server.xml.|TODO|
+|YARN |WebAppUtils uptakes the use of the credential provider API through the new method on Configuration called getPassword. This provides an alternative to storing the passwords in clear text within the ssl-server.xml file while maintaining backward compatibility.|TODO|
+|AWS <br/> S3/S3A |Uses Configuration.getPassword to get the S3 credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[AWS S3/S3A Usage](../../hadoop-aws/tools/hadoop-aws/index.html)|
+|Apache <br/> Accumulo|The trace.password property is used by the Tracer to authenticate with Accumulo and persist the traces in the trace table. The credential provider API is used to acquire the trace.password from a provider or from configuration for backward compatibility.|TODO|
+|Apache <br/> Slider |A capability has been added to Slider to prompt the user for needed passwords and store them using CredentialProvider so they can be retrieved by an app later.|TODO|
+|Apache <br/> Hive |Protection of the metastore password, SSL related passwords and JDO string password has been added through the use of the Credential Provider API|TODO|
+|Apache <br/> HBase |The HBase RESTServer is using the new Configuration.getPassword method so that the credential provider API will be checked first then fall back to clear text - when allowed.|TODO|
+|Apache <br/> Oozie |Protects SSL, email and JDBC passwords using the credential provider API.|TODO|
+|Apache <br/> Ranger |Protects database, trust and keystore passwords using the credential provider API.|TODO|
+
+### Credential Management
+
+#### The hadoop credential Command
+
+Usage: `hadoop credential <subcommand> [options]`
+
+See the command options detail in the [Commands Manual](CommandsManual.html#credential)
+
+Utilizing the credential command will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used.
+
+Example: `hadoop credential create ssl.server.keystore.password jceks://file/tmp/test.jceks`
+
+In order to indicate a particular provider type and location, the user must provide the `hadoop.security.credential.provider.path` configuration element in core-site.xml or use the command line option `-provider` on each of the credential management commands. This provider path is a comma-separated list of URLs that indicates the type and location of a list of providers that should be consulted. For example, the following path: `user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks` indicates that the current user's credentials file should be consulted through the User Provider, that the local file located at `/tmp/test.jceks` is a Java Keystore Provider and that the file located within HDFS at `nn1.example.com/my/path/test.jceks` is also a store for a Java Keystore Provider.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ab76580/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 00c5e65..cc737fa 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -66,6 +66,7 @@
<item name="Secure Mode" href="hadoop-project-dist/hadoop-common/SecureMode.html"/>
<item name="Service Level Authorization" href="hadoop-project-dist/hadoop-common/ServiceLevelAuth.html"/>
<item name="HTTP Authentication" href="hadoop-project-dist/hadoop-common/HttpAuthentication.html"/>
+ <item name="Credential Provider API" href="hadoop-project-dist/hadoop-common/CredentialProviderAPI.html"/>
<item name="Hadoop KMS" href="hadoop-kms/index.html"/>
<item name="Tracing" href="hadoop-project-dist/hadoop-common/Tracing.html"/>
<item name="Unix Shell Guide" href="hadoop-project-dist/hadoop-common/UnixShellGuide.html"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ab76580/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 73775c8..5e6eb95 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -154,6 +154,51 @@ If you do any of these: change your credentials immediately!
<description>AWS secret key. Omit for Role-based authentication.</description>
</property>
+#### Protecting the AWS Credentials in S3A
+
+To protect these credentials from prying eyes, it is recommended that you use
+the credential provider framework securely storing them and accessing them
+ through configuration. The following describes its use for AWS credentials
+in S3A FileSystem.
+
+For additional reading on the credential provider API see:
+[Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html).
+
+##### End to End Steps for Distcp and S3 with Credential Providers
+
+###### provision
+
+```
+% hadoop credential create fs.s3a.access.key -value 123
+ -provider localjceks://file/home/lmccay/aws.jceks
+```
+
+```
+% hadoop credential create fs.s3a.secret.key -value 456
+ -provider localjceks://file/home/lmccay/aws.jceks
+```
+
+###### configure core-site.xml or command line system property
+
+```
+<property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value>localjceks://file/home/lmccay/aws.jceks</value>
+ <description>Path to interrogate for protected credentials.</description>
+</property>
+```
+###### distcp
+
+```
+% hadoop distcp
+ [-D hadoop.security.credential.provider.path=localjceks://file/home/lmccay/aws.jceks]
+ hdfs://hostname:9001/user/lmccay/007020615 s3a://lmccay/
+```
+
+NOTE: You may optionally add the provider path property to the distcp command line instead of
+added job specific configuration to a generic coreÂsite.xml. The square brackets above illustrate
+this capability.
+
### Other properties
<property>
[47/50] [abbrv] hadoop git commit: YARN-4386.
refreshNodesGracefully() should send recommission event to active RMNodes
only. Contributed by Kuhu Shukla.
Posted by ar...@apache.org.
YARN-4386. refreshNodesGracefully() should send recommission event to active RMNodes only. Contributed by Kuhu Shukla.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fab8854
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fab8854
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fab8854
Branch: refs/heads/HDFS-1312
Commit: 3fab88540f079591e5dae1c9184f8b26bb843427
Parents: 5e7d4d5
Author: Junping Du <ju...@apache.org>
Authored: Mon Feb 22 07:04:19 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Mon Feb 22 07:04:19 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../resourcemanager/NodesListManager.java | 3 +-
.../TestResourceTrackerService.java | 35 ++++++++++++++++++++
3 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fab8854/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e39917f..9b57666 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1433,6 +1433,9 @@ Release 2.8.0 - UNRELEASED
YARN-4654. Yarn node label CLI should parse "=" correctly when trying to
remove all labels on a node. (Naganarasimha G R via rohithsharmaks)
+ YARN-4386. refreshNodesGracefully() should send recommission event to active
+ RMNodes only. (Kuhu Shukla via junping_du)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fab8854/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index e6251fe..4cdf67a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -400,8 +400,7 @@ public class NodesListManager extends CompositeService implements
new RMNodeEvent(nodeId, RMNodeEventType.GRACEFUL_DECOMMISSION));
} else {
// Recommissioning the nodes
- if (entry.getValue().getState() == NodeState.DECOMMISSIONING
- || entry.getValue().getState() == NodeState.DECOMMISSIONED) {
+ if (entry.getValue().getState() == NodeState.DECOMMISSIONING) {
this.rmContext.getDispatcher().getEventHandler()
.handle(new RMNodeEvent(nodeId, RMNodeEventType.RECOMMISSION));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fab8854/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index e0fd9ab..9ed79a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -1269,6 +1269,41 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm1.stop();
}
+ @Test
+ public void testIncorrectRecommission() throws Exception {
+ //Check decommissioned node not get recommissioned with graceful refresh
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ File excludeHostFile =
+ new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
+ writeToHostsFile(excludeHostFile, "host3", "host2");
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ excludeHostFile.getAbsolutePath());
+ writeToHostsFile(hostFile, "host1", "host2");
+ writeToHostsFile(excludeHostFile, "host1");
+ rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.drainEvents();
+ nm1.nodeHeartbeat(true);
+ rm.drainEvents();
+ Assert.assertTrue("Node " + nm1.getNodeId().getHost() +
+ " should be Decommissioned", rm.getRMContext()
+ .getInactiveRMNodes().get(nm1.getNodeId()).getState() == NodeState
+ .DECOMMISSIONED);
+ writeToHostsFile(excludeHostFile, "");
+ rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.drainEvents();
+ Assert.assertTrue("Node " + nm1.getNodeId().getHost() +
+ " should be Decommissioned", rm.getRMContext()
+ .getInactiveRMNodes().get(nm1.getNodeId()).getState() == NodeState
+ .DECOMMISSIONED);
+ rm.stop();
+ }
+
private void writeToHostsFile(String... hosts) throws IOException {
writeToHostsFile(hostFile, hosts);
}
[38/50] [abbrv] hadoop git commit: Support additional compression
levels for GzipCodec. Contributed by Ravi Mutyala.
Posted by ar...@apache.org.
Support additional compression levels for GzipCodec. Contributed by Ravi Mutyala.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18f9b77a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18f9b77a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18f9b77a
Branch: refs/heads/HDFS-1312
Commit: 18f9b77a321b225677ce23c503b41d21478fc4a7
Parents: 6546d9e
Author: Junping Du <ju...@apache.org>
Authored: Fri Feb 19 14:21:25 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Fri Feb 19 14:21:25 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/io/compress/zlib/ZlibCompressor.java | 42 +++++++++++++++++++-
.../zlib/TestZlibCompressorDecompressor.java | 25 ++++++++++++
3 files changed, 69 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18f9b77a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bf84d7a..e53c001 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1737,6 +1737,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12413. AccessControlList should avoid calling getGroupNames in
isUserInList with empty groups. (Zhihai Xu via cnauroth)
+ HADOOP-12794. Support additional compression levels for GzipCodec.
+ (Ravi Mutyala via junping_du)
+
OPTIMIZATIONS
HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18f9b77a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
index b955044..24d98a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
@@ -67,7 +67,47 @@ public class ZlibCompressor implements Compressor {
* Compression level for fastest compression.
*/
BEST_SPEED (1),
-
+
+ /**
+ * Compression level 2.
+ */
+ TWO (2),
+
+
+ /**
+ * Compression level 3.
+ */
+ THREE (3),
+
+
+ /**
+ * Compression level 4.
+ */
+ FOUR (4),
+
+
+ /**
+ * Compression level 5.
+ */
+ FIVE (5),
+
+
+ /**
+ * Compression level 6.
+ */
+ SIX (6),
+
+ /**
+ * Compression level 7.
+ */
+ SEVEN (7),
+
+ /**
+ * Compression level 8.
+ */
+ EIGHT (8),
+
+
/**
* Compression level for best compression.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18f9b77a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
index 6a4fc60..7e2ab24 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
@@ -122,6 +122,31 @@ public class TestZlibCompressorDecompressor {
}
@Test
+ public void testZlibCompressorDecompressorWithCompressionLevels() {
+ Configuration conf = new Configuration();
+ conf.set("zlib.compress.level","FOUR");
+ if (ZlibFactory.isNativeZlibLoaded(conf)) {
+ byte[] rawData;
+ int tryNumber = 5;
+ int BYTE_SIZE = 10 * 1024;
+ Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
+ Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
+ rawData = generate(BYTE_SIZE);
+ try {
+ for (int i = 0; i < tryNumber; i++)
+ compressDecompressZlib(rawData, (ZlibCompressor) zlibCompressor,
+ (ZlibDecompressor) zlibDecompressor);
+ zlibCompressor.reinit(conf);
+ } catch (Exception ex) {
+ fail("testZlibCompressorDecompressorWithConfiguration ex error " + ex);
+ }
+ } else {
+ assertTrue("ZlibFactory is using native libs against request",
+ ZlibFactory.isNativeZlibLoaded(conf));
+ }
+ }
+
+ @Test
public void testZlibCompressDecompress() {
byte[] rawData = null;
int rawDataSize = 0;
[41/50] [abbrv] hadoop git commit: HDFS-9818. Correctly handle EC
reconstruction work caused by not enough racks. Contributed by Jing Zhao.
Posted by ar...@apache.org.
HDFS-9818. Correctly handle EC reconstruction work caused by not enough racks. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e54cc293
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e54cc293
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e54cc293
Branch: refs/heads/HDFS-1312
Commit: e54cc2931262bf49682a8323da9811976218c03b
Parents: 6eae433
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Feb 19 19:02:23 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Feb 19 19:02:23 2016 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedInputStream.java | 7 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 40 ++--
.../BlockPlacementPolicyDefault.java | 5 +-
.../BlockPlacementPolicyRackFaultTolerant.java | 10 +-
.../BlockPlacementStatusDefault.java | 10 +-
.../BlockReconstructionWork.java | 5 +
.../blockmanagement/ErasureCodingWork.java | 106 ++++++++-
.../server/blockmanagement/ReplicationWork.java | 5 +
.../erasurecode/ErasureCodingWorker.java | 25 ++-
.../TestBlocksWithNotEnoughRacks.java | 12 +-
...constructStripedBlocksWithRackAwareness.java | 213 +++++++++++++++++++
12 files changed, 388 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index d15e536..3483255 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -260,7 +260,12 @@ public class DFSStripedInputStream extends DFSInputStream {
private void closeReader(BlockReaderInfo readerInfo) {
if (readerInfo != null) {
-// IOUtils.cleanup(null, readerInfo.reader);
+ if (readerInfo.reader != null) {
+ try {
+ readerInfo.reader.close();
+ } catch (Throwable ignored) {
+ }
+ }
readerInfo.skip();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a377243..1d0379c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -432,6 +432,9 @@ Trunk (Unreleased)
HDFS-9794. Streamer threads may leak if failure happens when closing the
striped outputstream. (jing9)
+ HDFS-9818. Correctly handle EC reconstruction work caused by not enough
+ racks. (jing9)
+
BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
HDFS-7347. Configurable erasure coding policy for individual files and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 67f5026..cc52b6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1628,7 +1628,7 @@ public class BlockManager implements BlockStatsMXBean {
for (int i = 0 ; i < liveBlockIndices.size(); i++) {
indices[i] = liveBlockIndices.get(i);
}
- return new ErasureCodingWork(block, bc, srcNodes,
+ return new ErasureCodingWork(getBlockPoolId(), block, bc, srcNodes,
containingNodes, liveReplicaNodes, additionalReplRequired,
priority, indices);
} else {
@@ -1638,6 +1638,16 @@ public class BlockManager implements BlockStatsMXBean {
}
}
+ private boolean isInNewRack(DatanodeDescriptor[] srcs,
+ DatanodeDescriptor target) {
+ for (DatanodeDescriptor src : srcs) {
+ if (src.getNetworkLocation().equals(target.getNetworkLocation())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private boolean validateReconstructionWork(BlockReconstructionWork rw) {
BlockInfo block = rw.getBlock();
int priority = rw.getPriority();
@@ -1665,31 +1675,14 @@ public class BlockManager implements BlockStatsMXBean {
DatanodeStorageInfo[] targets = rw.getTargets();
if ( (numReplicas.liveReplicas() >= requiredReplication) &&
(!isPlacementPolicySatisfied(block)) ) {
- if (rw.getSrcNodes()[0].getNetworkLocation().equals(
- targets[0].getDatanodeDescriptor().getNetworkLocation())) {
- //No use continuing, unless a new rack in this case
+ if (!isInNewRack(rw.getSrcNodes(), targets[0].getDatanodeDescriptor())) {
+ // No use continuing, unless a new rack in this case
return false;
}
}
- // Add block to the to be reconstructed list
- if (block.isStriped()) {
- assert rw instanceof ErasureCodingWork;
- assert rw.getTargets().length > 0;
- assert pendingNum == 0 : "Should wait the previous reconstruction"
- + " to finish";
- final ErasureCodingPolicy ecPolicy =
- ((BlockInfoStriped) block).getErasureCodingPolicy();
- assert ecPolicy != null;
-
- rw.getTargets()[0].getDatanodeDescriptor().addBlockToBeErasureCoded(
- new ExtendedBlock(getBlockPoolId(), block),
- rw.getSrcNodes(), rw.getTargets(),
- ((ErasureCodingWork) rw).getLiveBlockIndicies(), ecPolicy);
- } else {
- rw.getSrcNodes()[0].addBlockToBeReplicated(block, targets);
- }
-
+ // Add block to the datanode's task list
+ rw.addTaskToDatanode();
DatanodeStorageInfo.incrementBlocksScheduled(targets);
// Move the block-replication into a "pending" state.
@@ -3973,7 +3966,8 @@ public class BlockManager implements BlockStatsMXBean {
.getPolicy(storedBlock.isStriped());
int numReplicas = storedBlock.isStriped() ? ((BlockInfoStriped) storedBlock)
.getRealDataBlockNum() : storedBlock.getReplication();
- return placementPolicy.verifyBlockPlacement(locs, numReplicas).isPlacementPolicySatisfied();
+ return placementPolicy.verifyBlockPlacement(locs, numReplicas)
+ .isPlacementPolicySatisfied();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index ee891a5..e1a47ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -901,7 +901,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
locs = DatanodeDescriptor.EMPTY_ARRAY;
if (!clusterMap.hasClusterEverBeenMultiRack()) {
// only one rack
- return new BlockPlacementStatusDefault(1, 1);
+ return new BlockPlacementStatusDefault(1, 1, 1);
}
int minRacks = 2;
minRacks = Math.min(minRacks, numberOfReplicas);
@@ -910,7 +910,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
Set<String> racks = new TreeSet<>();
for (DatanodeInfo dn : locs)
racks.add(dn.getNetworkLocation());
- return new BlockPlacementStatusDefault(racks.size(), minRacks);
+ return new BlockPlacementStatusDefault(racks.size(), minRacks,
+ clusterMap.getNumOfRacks());
}
/**
* Decide whether deleting the specified replica of the block still makes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
index c803b97..c0d981c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
@@ -160,14 +160,16 @@ public class BlockPlacementPolicyRackFaultTolerant extends BlockPlacementPolicyD
locs = DatanodeDescriptor.EMPTY_ARRAY;
if (!clusterMap.hasClusterEverBeenMultiRack()) {
// only one rack
- return new BlockPlacementStatusDefault(1, 1);
+ return new BlockPlacementStatusDefault(1, 1, 1);
}
// 1. Check that all locations are different.
// 2. Count locations on different racks.
- Set<String> racks = new TreeSet<String>();
- for (DatanodeInfo dn : locs)
+ Set<String> racks = new TreeSet<>();
+ for (DatanodeInfo dn : locs) {
racks.add(dn.getNetworkLocation());
- return new BlockPlacementStatusDefault(racks.size(), numberOfReplicas);
+ }
+ return new BlockPlacementStatusDefault(racks.size(), numberOfReplicas,
+ clusterMap.getNumOfRacks());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
index 0b8b965..75bb65d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java
@@ -21,15 +21,18 @@ public class BlockPlacementStatusDefault implements BlockPlacementStatus {
private int requiredRacks = 0;
private int currentRacks = 0;
+ private final int totalRacks;
- public BlockPlacementStatusDefault(int currentRacks, int requiredRacks){
+ public BlockPlacementStatusDefault(int currentRacks, int requiredRacks,
+ int totalRacks){
this.requiredRacks = requiredRacks;
this.currentRacks = currentRacks;
+ this.totalRacks = totalRacks;
}
@Override
public boolean isPlacementPolicySatisfied() {
- return requiredRacks <= currentRacks;
+ return requiredRacks <= currentRacks || currentRacks >= totalRacks;
}
@Override
@@ -38,7 +41,8 @@ public class BlockPlacementStatusDefault implements BlockPlacementStatus {
return null;
}
return "Block should be additionally replicated on " +
- (requiredRacks - currentRacks) + " more rack(s).";
+ (requiredRacks - currentRacks) +
+ " more rack(s). Total number of racks in the cluster: " + totalRacks;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
index df9c164..c1998ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
@@ -108,4 +108,9 @@ abstract class BlockReconstructionWork {
abstract void chooseTargets(BlockPlacementPolicy blockplacement,
BlockStoragePolicySuite storagePolicySuite,
Set<Node> excludedNodes);
+
+ /**
+ * add reconstruction task into a source datanode
+ */
+ abstract void addTaskToDatanode();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
index 85a25d5..38ad324 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
@@ -17,15 +17,23 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.net.Node;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
class ErasureCodingWork extends BlockReconstructionWork {
private final byte[] liveBlockIndicies;
+ private final String blockPoolId;
- public ErasureCodingWork(BlockInfo block,
+ public ErasureCodingWork(String blockPoolId, BlockInfo block,
BlockCollection bc,
DatanodeDescriptor[] srcNodes,
List<DatanodeDescriptor> containingNodes,
@@ -34,6 +42,7 @@ class ErasureCodingWork extends BlockReconstructionWork {
int priority, byte[] liveBlockIndicies) {
super(block, bc, srcNodes, containingNodes,
liveReplicaStorages, additionalReplRequired, priority);
+ this.blockPoolId = blockPoolId;
this.liveBlockIndicies = liveBlockIndicies;
BlockManager.LOG.debug("Creating an ErasureCodingWork to {} reconstruct ",
block);
@@ -47,15 +56,92 @@ class ErasureCodingWork extends BlockReconstructionWork {
void chooseTargets(BlockPlacementPolicy blockplacement,
BlockStoragePolicySuite storagePolicySuite,
Set<Node> excludedNodes) {
- try {
- // TODO: new placement policy for EC considering multiple writers
- DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
- getBc().getName(), getAdditionalReplRequired(), getSrcNodes()[0],
- getLiveReplicaStorages(), false, excludedNodes,
- getBlock().getNumBytes(),
- storagePolicySuite.getPolicy(getBc().getStoragePolicyID()));
- setTargets(chosenTargets);
- } finally {
+ // TODO: new placement policy for EC considering multiple writers
+ DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
+ getBc().getName(), getAdditionalReplRequired(), getSrcNodes()[0],
+ getLiveReplicaStorages(), false, excludedNodes,
+ getBlock().getNumBytes(),
+ storagePolicySuite.getPolicy(getBc().getStoragePolicyID()));
+ setTargets(chosenTargets);
+ }
+
+ /**
+ * @return true if the current source nodes cover all the internal blocks.
+ * I.e., we only need to have more racks.
+ */
+ private boolean hasAllInternalBlocks() {
+ final BlockInfoStriped block = (BlockInfoStriped) getBlock();
+ if (getSrcNodes().length < block.getRealTotalBlockNum()) {
+ return false;
+ }
+ BitSet bitSet = new BitSet(block.getTotalBlockNum());
+ for (byte index : liveBlockIndicies) {
+ bitSet.set(index);
+ }
+ for (int i = 0; i < block.getRealDataBlockNum(); i++) {
+ if (!bitSet.get(i)) {
+ return false;
+ }
+ }
+ for (int i = block.getDataBlockNum(); i < block.getTotalBlockNum(); i++) {
+ if (!bitSet.get(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * We have all the internal blocks but not enough racks. Thus we do not need
+ * to do decoding but only simply make an extra copy of an internal block. In
+ * this scenario, use this method to choose the source datanode for simple
+ * replication.
+ * @return The index of the source datanode.
+ */
+ private int chooseSource4SimpleReplication() {
+ Map<String, List<Integer>> map = new HashMap<>();
+ for (int i = 0; i < getSrcNodes().length; i++) {
+ final String rack = getSrcNodes()[i].getNetworkLocation();
+ List<Integer> dnList = map.get(rack);
+ if (dnList == null) {
+ dnList = new ArrayList<>();
+ map.put(rack, dnList);
+ }
+ dnList.add(i);
+ }
+ List<Integer> max = null;
+ for (Map.Entry<String, List<Integer>> entry : map.entrySet()) {
+ if (max == null || entry.getValue().size() > max.size()) {
+ max = entry.getValue();
+ }
+ }
+ assert max != null;
+ return max.get(0);
+ }
+
+ @Override
+ void addTaskToDatanode() {
+ assert getTargets().length > 0;
+ BlockInfoStriped stripedBlk = (BlockInfoStriped) getBlock();
+
+ // if we already have all the internal blocks, but not enough racks,
+ // we only need to replicate one internal block to a new rack
+ if (hasAllInternalBlocks()) {
+ int sourceIndex = chooseSource4SimpleReplication();
+ final byte blockIndex = liveBlockIndicies[sourceIndex];
+ final DatanodeDescriptor source = getSrcNodes()[sourceIndex];
+ final long internBlkLen = StripedBlockUtil.getInternalBlockLength(
+ stripedBlk.getNumBytes(), stripedBlk.getCellSize(),
+ stripedBlk.getDataBlockNum(), blockIndex);
+ final Block targetBlk = new Block(
+ stripedBlk.getBlockId() + blockIndex, internBlkLen,
+ stripedBlk.getGenerationStamp());
+ source.addBlockToBeReplicated(targetBlk, getTargets());
+ } else {
+ getTargets()[0].getDatanodeDescriptor().addBlockToBeErasureCoded(
+ new ExtendedBlock(blockPoolId, stripedBlk),
+ getSrcNodes(), getTargets(), getLiveBlockIndicies(),
+ stripedBlk.getErasureCodingPolicy());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
index b44b9b1..24601a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
@@ -53,4 +53,9 @@ class ReplicationWork extends BlockReconstructionWork {
getSrcNodes()[0].decrementPendingReplicationWithoutTargets();
}
}
+
+ @Override
+ void addTaskToDatanode() {
+ getSrcNodes()[0].addBlockToBeReplicated(getBlock(), getTargets());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index b08aa2e..1017e1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -177,8 +177,14 @@ public final class ErasureCodingWorker {
Collection<BlockECReconstructionInfo> ecTasks) {
for (BlockECReconstructionInfo reconstructionInfo : ecTasks) {
try {
- EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL
- .submit(new ReconstructAndTransferBlock(reconstructionInfo));
+ ReconstructAndTransferBlock task =
+ new ReconstructAndTransferBlock(reconstructionInfo);
+ if (task.hasValidTargets()) {
+ EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL.submit(task);
+ } else {
+ LOG.warn("No missing internal block. Skip reconstruction for task:{}",
+ reconstructionInfo);
+ }
} catch (Throwable e) {
LOG.warn("Failed to reconstruct striped block {}",
reconstructionInfo.getExtendedBlock().getLocalBlock(), e);
@@ -292,6 +298,7 @@ public final class ErasureCodingWorker {
private final CompletionService<Void> readService =
new ExecutorCompletionService<>(
EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL);
+ private final boolean hasValidTargets;
ReconstructAndTransferBlock(BlockECReconstructionInfo reconstructionInfo) {
ErasureCodingPolicy ecPolicy = reconstructionInfo
@@ -339,10 +346,14 @@ public final class ErasureCodingWorker {
seqNo4Targets[i] = 0;
}
- getTargetIndices();
+ hasValidTargets = getTargetIndices();
cachingStrategy = CachingStrategy.newDefaultStrategy();
}
+ boolean hasValidTargets() {
+ return hasValidTargets;
+ }
+
private ByteBuffer allocateBuffer(int length) {
return ByteBuffer.allocate(length);
}
@@ -505,24 +516,30 @@ public final class ErasureCodingWorker {
}
}
- private void getTargetIndices() {
+ /**
+ * @return true if there is valid target for reconstruction
+ */
+ private boolean getTargetIndices() {
BitSet bitset = new BitSet(dataBlkNum + parityBlkNum);
for (int i = 0; i < sources.length; i++) {
bitset.set(liveIndices[i]);
}
int m = 0;
int k = 0;
+ boolean hasValidTarget = false;
for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
if (!bitset.get(i)) {
if (getBlockLen(blockGroup, i) > 0) {
if (m < targets.length) {
targetIndices[m++] = (short)i;
+ hasValidTarget = true;
}
} else {
zeroStripeIndices[k++] = (short)i;
}
}
}
+ return hasValidTarget;
}
/** the reading length should not exceed the length for reconstruction. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
index ea994a2..d91155a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
@@ -285,7 +285,7 @@ public class TestBlocksWithNotEnoughRacks {
final DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
try {
- // Create a file with one block with a replication factor of 2
+ // Create a file with one block with a replication factor of 3
final FileSystem fs = cluster.getFileSystem();
DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
@@ -315,8 +315,9 @@ public class TestBlocksWithNotEnoughRacks {
dm.removeDatanode(dnId);
// Make sure we have enough live replicas even though we are
- // short one rack and therefore need one replica
- DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 1);
+ // short one rack. The cluster now has only 1 rack thus we just make sure
+ // we still have 3 replicas.
+ DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
} finally {
cluster.shutdown();
}
@@ -357,9 +358,8 @@ public class TestBlocksWithNotEnoughRacks {
// The block gets re-replicated to another datanode so it has a
// sufficient # replicas, but not across racks, so there should
- // be 1 rack, and 1 needed replica (even though there are 2 hosts
- // available and only 2 replicas required).
- DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 1);
+ // be 1 rack.
+ DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
// Start the "failed" datanode, which has a replica so the block is
// now over-replicated and therefore a replica should be removed but
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e54cc293/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
new file mode 100644
index 0000000..2164957
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
+
+public class TestReconstructStripedBlocksWithRackAwareness {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ TestReconstructStripedBlocksWithRackAwareness.class);
+
+ static {
+ GenericTestUtils.setLogLevel(BlockPlacementPolicy.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(BlockManager.blockLog, Level.ALL);
+ }
+
+ private static final String[] hosts = new String[]{"host1", "host2", "host3",
+ "host4", "host5", "host6", "host7", "host8", "host9", "host10"};
+ private static final String[] racks = new String[]{"/r1", "/r1", "/r2", "/r2",
+ "/r3", "/r3", "/r4", "/r4", "/r5", "/r6"};
+ private static final List<String> singleNodeRacks = Arrays.asList("host9", "host10");
+ private static final short blockNum = (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private FSNamesystem fsn;
+ private BlockManager bm;
+
+ @Before
+ public void setup() throws Exception {
+ final HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ false);
+
+ cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts)
+ .numDataNodes(hosts.length).build();
+ cluster.waitActive();
+
+ fsn = cluster.getNamesystem();
+ bm = fsn.getBlockManager();
+
+ fs = cluster.getFileSystem();
+ fs.setErasureCodingPolicy(new Path("/"), null);
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * When there are all the internal blocks available but they are not placed on
+ * enough racks, NameNode should avoid normal decoding reconstruction but copy
+ * an internal block to a new rack.
+ *
+ * In this test, we first need to create a scenario that a striped block has
+ * all the internal blocks but distributed in <6 racks. Then we check if the
+ * replication monitor can correctly schedule the reconstruction work for it.
+ *
+ * For the 9 internal blocks + 5 racks setup, the test does the following:
+ * 1. create a 6 rack cluster with 10 datanodes, where there are 2 racks only
+ * containing 1 datanodes each
+ * 2. for a striped block with 9 internal blocks, there must be one internal
+ * block locating in a single-node rack. find this node and stop it
+ * 3. namenode will trigger reconstruction for the block and since the cluster
+ * has only 5 racks remaining, after the reconstruction we have 9 internal
+ * blocks distributed in 5 racks.
+ * 4. we bring the datanode back, now the cluster has 6 racks again
+ * 5. let the datanode call reportBadBlock, this will make the namenode to
+ * check if the striped block is placed in >= 6 racks, and the namenode will
+ * put the block into the under-replicated queue
+ * 6. now we can check if the replication monitor works as expected
+ */
+ @Test
+ public void testReconstructForNotEnoughRacks() throws Exception {
+ final Path file = new Path("/foo");
+ DFSTestUtil.createFile(fs, file,
+ BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
+ Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
+
+ final INodeFile fileNode = fsn.getFSDirectory()
+ .getINode4Write(file.toString()).asFile();
+ BlockInfoStriped blockInfo = (BlockInfoStriped) fileNode.getLastBlock();
+
+ // find the internal block located in the single node rack
+ Block internalBlock = null;
+ String hostToStop = null;
+ for (DatanodeStorageInfo storage : blockInfo.storages) {
+ if (singleNodeRacks.contains(storage.getDatanodeDescriptor().getHostName())) {
+ hostToStop = storage.getDatanodeDescriptor().getHostName();
+ internalBlock = blockInfo.getBlockOnStorage(storage);
+ }
+ }
+ Assert.assertNotNull(internalBlock);
+ Assert.assertNotNull(hostToStop);
+
+ // delete the block on the chosen datanode
+ cluster.corruptBlockOnDataNodesByDeletingBlockFile(
+ new ExtendedBlock(bm.getBlockPoolId(), internalBlock));
+
+ // stop the chosen datanode
+ MiniDFSCluster.DataNodeProperties dnProp = null;
+ for (int i = 0; i < cluster.getDataNodes().size(); i++) {
+ DataNode dn = cluster.getDataNodes().get(i);
+ if (dn.getDatanodeId().getHostName().equals(hostToStop)) {
+ dnProp = cluster.stopDataNode(i);
+ cluster.setDataNodeDead(dn.getDatanodeId());
+ LOG.info("stop datanode " + dn.getDatanodeId().getHostName());
+ }
+ }
+ NetworkTopology topology = bm.getDatanodeManager().getNetworkTopology();
+ Assert.assertEquals(5, topology.getNumOfRacks());
+
+ // make sure the reconstruction work can finish
+ // now we have 9 internal blocks in 5 racks
+ DFSTestUtil.waitForReplication(fs, file, blockNum, 15 * 1000);
+
+ // we now should have 9 internal blocks distributed in 5 racks
+ Set<String> rackSet = new HashSet<>();
+ for (DatanodeStorageInfo storage : blockInfo.storages) {
+ rackSet.add(storage.getDatanodeDescriptor().getNetworkLocation());
+ }
+ Assert.assertEquals(5, rackSet.size());
+
+ // restart the stopped datanode
+ cluster.restartDataNode(dnProp);
+ cluster.waitActive();
+
+ // make sure we have 6 racks again
+ topology = bm.getDatanodeManager().getNetworkTopology();
+ Assert.assertEquals(hosts.length, topology.getNumOfLeaves());
+ Assert.assertEquals(6, topology.getNumOfRacks());
+
+ // pause all the heartbeats
+ DataNode badDn = null;
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ if (dn.getDatanodeId().getHostName().equals(hostToStop)) {
+ badDn = dn;
+ }
+ }
+ assert badDn != null;
+ // let the DN report the bad block, so that the namenode will put the block
+ // into under-replicated queue. note that the block still has 9 internal
+ // blocks but in 5 racks
+ badDn.reportBadBlocks(new ExtendedBlock(bm.getBlockPoolId(), internalBlock));
+
+ // check if replication monitor correctly schedule the replication work
+ boolean scheduled = false;
+ for (int i = 0; i < 5; i++) { // retry 5 times
+ for (DatanodeStorageInfo storage : blockInfo.storages) {
+ if (storage != null) {
+ DatanodeDescriptor dn = storage.getDatanodeDescriptor();
+ Assert.assertEquals(0, dn.getNumberOfBlocksToBeErasureCoded());
+ if (dn.getNumberOfBlocksToBeReplicated() == 1) {
+ scheduled = true;
+ }
+ }
+ }
+ if (scheduled) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ Assert.assertTrue(scheduled);
+ }
+}
[49/50] [abbrv] hadoop git commit: HDFS-9671. DiskBalancer:
SubmitPlan implementation. (Contributed by Anu Engineer)
Posted by ar...@apache.org.
HDFS-9671. DiskBalancer: SubmitPlan implementation. (Contributed by Anu Engineer)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f72fcca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f72fcca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f72fcca
Branch: refs/heads/HDFS-1312
Commit: 3f72fcca6d5a52b42b8d6278370baae6ddc53fbf
Parents: a0efd53
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Feb 22 11:45:51 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Feb 22 11:45:51 2016 -0800
----------------------------------------------------------------------
.../hdfs/protocol/ClientDatanodeProtocol.java | 4 +-
.../ClientDatanodeProtocolTranslatorPB.java | 10 +-
.../server/datanode/DiskBalancerWorkItem.java | 160 ++++++
.../server/datanode/DiskBalancerWorkStatus.java | 87 +++
.../hadoop/hdfs/server/datanode/WorkStatus.java | 85 ---
.../hadoop-hdfs/HDFS-1312_CHANGES.txt | 7 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
...tDatanodeProtocolServerSideTranslatorPB.java | 4 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 56 +-
.../hdfs/server/datanode/DiskBalancer.java | 542 +++++++++++++++++++
.../diskbalancer/DiskBalancerConstants.java | 9 +
.../diskbalancer/DiskBalancerException.java | 98 ++++
.../diskbalancer/DiskbalancerException.java | 86 ---
.../datamodel/DiskBalancerCluster.java | 14 +
.../diskbalancer/TestDiskBalancerRPC.java | 28 +-
15 files changed, 984 insertions(+), 211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
index 706bc04..cf29bc5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSelector;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.hadoop.hdfs.server.datanode.WorkStatus;
+import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus;
/** An client-datanode protocol for block recovery
*/
@@ -175,7 +175,7 @@ public interface ClientDatanodeProtocol {
/**
* Gets the status of an executing diskbalancer Plan.
*/
- WorkStatus queryDiskBalancerPlan() throws IOException;
+ DiskBalancerWorkStatus queryDiskBalancerPlan() throws IOException;
/**
* Gets a run-time configuration value from running diskbalancer instance.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
index 04f21e0..524d127 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.QueryP
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DiskBalancerSettingRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DiskBalancerSettingResponseProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.server.datanode.WorkStatus;
+import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
@@ -333,8 +333,7 @@ public class ClientDatanodeProtocolTranslatorPB implements
* to zero allows datanode to use the value defined in
* configration.
* @param plan - Actual plan.
- * @return Success or throws Exception.
- * @throws Exception
+ * @throws IOException
*/
@Override
public void submitDiskBalancerPlan(String planID, long planVersion,
@@ -375,13 +374,14 @@ public class ClientDatanodeProtocolTranslatorPB implements
* Gets the status of an executing diskbalancer Plan.
*/
@Override
- public WorkStatus queryDiskBalancerPlan() throws IOException {
+ public DiskBalancerWorkStatus queryDiskBalancerPlan() throws IOException {
try {
QueryPlanStatusRequestProto request =
QueryPlanStatusRequestProto.newBuilder().build();
QueryPlanStatusResponseProto response =
rpcProxy.queryDiskBalancerPlan(NULL_CONTROLLER, request);
- return new WorkStatus(response.hasResult() ? response.getResult() : 0,
+ return new DiskBalancerWorkStatus(response.hasResult() ?
+ response.getResult() : 0,
response.hasPlanID() ? response.getPlanID() : null,
response.hasStatus() ? response.getStatus() : null,
response.hasCurrentStatus() ? response.getCurrentStatus() : null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
new file mode 100644
index 0000000..11730e2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+
+/**
+ * Keeps track of how much work has finished.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DiskBalancerWorkItem {
+ private final long bytesToCopy;
+ private long bytesCopied;
+ private long errorCount;
+ private String errMsg;
+ private long blocksCopied;
+
+ /**
+ * Constructs a DiskBalancerWorkItem.
+ *
+ * @param bytesToCopy - Total bytes to copy from a disk
+ * @param bytesCopied - Copied So far.
+ */
+ public DiskBalancerWorkItem(long bytesToCopy, long bytesCopied) {
+ this.bytesToCopy = bytesToCopy;
+ this.bytesCopied = bytesCopied;
+ }
+
+ /**
+ * Reads a DiskBalancerWorkItem Object from a Json String.
+ *
+ * @param json - Json String.
+ * @return DiskBalancerWorkItem Object
+ * @throws IOException
+ */
+ public static DiskBalancerWorkItem parseJson(String json) throws IOException {
+ Preconditions.checkNotNull(json);
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.readValue(json, DiskBalancerWorkItem.class);
+ }
+
+ /**
+ * Gets the error message.
+ */
+ public String getErrMsg() {
+ return errMsg;
+ }
+
+ /**
+ * Sets the error message.
+ *
+ * @param errMsg - Msg.
+ */
+ public void setErrMsg(String errMsg) {
+ this.errMsg = errMsg;
+ }
+
+ /**
+ * Returns the number of errors encountered.
+ *
+ * @return long
+ */
+ public long getErrorCount() {
+ return errorCount;
+ }
+
+ /**
+ * Incs Error Count.
+ */
+ public void incErrorCount() {
+ this.errorCount++;
+ }
+
+ /**
+ * Returns bytes copied so far.
+ *
+ * @return long
+ */
+ public long getBytesCopied() {
+ return bytesCopied;
+ }
+
+ /**
+ * Sets bytes copied so far.
+ *
+ * @param bytesCopied - long
+ */
+ public void setBytesCopied(long bytesCopied) {
+ this.bytesCopied = bytesCopied;
+ }
+
+ /**
+ * Increments bytesCopied by delta.
+ *
+ * @param delta - long
+ */
+ public void incCopiedSoFar(long delta) {
+ this.bytesCopied += delta;
+ }
+
+ /**
+ * Returns bytes to copy.
+ *
+ * @return - long
+ */
+ public long getBytesToCopy() {
+ return bytesToCopy;
+ }
+
+ /**
+ * Returns number of blocks copied for this DiskBalancerWorkItem.
+ *
+ * @return long count of blocks.
+ */
+ public long getBlocksCopied() {
+ return blocksCopied;
+ }
+
+ /**
+ * increments the number of blocks copied.
+ */
+ public void incBlocksCopied() {
+ blocksCopied++;
+ }
+
+ /**
+ * returns a serialized json string.
+ *
+ * @return String - json
+ * @throws IOException
+ */
+ public String toJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(this);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
new file mode 100644
index 0000000..6b29ce8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Helper class that reports how much work has has been done by the node.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DiskBalancerWorkStatus {
+ private final int result;
+ private final String planID;
+ private final String status;
+ private final String currentState;
+
+ /**
+ * Constructs a workStatus Object.
+ *
+ * @param result - int
+ * @param planID - Plan ID
+ * @param status - Current Status
+ * @param currentState - Current State
+ */
+ public DiskBalancerWorkStatus(int result, String planID, String status,
+ String currentState) {
+ this.result = result;
+ this.planID = planID;
+ this.status = status;
+ this.currentState = currentState;
+ }
+
+ /**
+ * Returns result.
+ *
+ * @return long
+ */
+ public int getResult() {
+ return result;
+ }
+
+ /**
+ * Returns planID.
+ *
+ * @return String
+ */
+ public String getPlanID() {
+ return planID;
+ }
+
+ /**
+ * Returns Status.
+ *
+ * @return String
+ */
+ public String getStatus() {
+ return status;
+ }
+
+ /**
+ * Gets current Status.
+ *
+ * @return - Json String
+ */
+ public String getCurrentState() {
+ return currentState;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java
deleted file mode 100644
index 259a311..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/WorkStatus.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Helper class that reports how much work has has been done by the node.
- */
-@InterfaceAudience.Private
-public class WorkStatus {
- private int result;
- private String planID;
- private String status;
- private String currentState;
-
- /**
- * Constructs a workStatus Object.
- *
- * @param result - int
- * @param planID - Plan ID
- * @param status - Current Status
- * @param currentState - Current State
- */
- public WorkStatus(int result, String planID, String status,
- String currentState) {
- this.result = result;
- this.planID = planID;
- this.status = status;
- this.currentState = currentState;
- }
-
- /**
- * Returns result.
- *
- * @return long
- */
- public int getResult() {
- return result;
- }
-
- /**
- * Returns planID.
- *
- * @return String
- */
- public String getPlanID() {
- return planID;
- }
-
- /**
- * Returns Status.
- *
- * @return String
- */
- public String getStatus() {
- return status;
- }
-
- /**
- * Gets current Status.
- *
- * @return - Json String
- */
- public String getCurrentState() {
- return currentState;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
index d3bdedf..27de7d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/HDFS-1312_CHANGES.txt
@@ -21,6 +21,9 @@ HDFS-1312 Change Log
HDFS-9645. DiskBalancer: Add Query RPC. (Anu Engineer via Arpit Agarwal)
- HDFS-9647. DiskBalancer: Add getRuntimeSettings. (Anu Engineer
- via Arpit Agarwal)
+ HDFS-9647. DiskBalancer: Add getRuntimeSettings. (Anu Engineer via
+ Arpit Agarwal)
+
+ HDFS-9671skBalancer: SubmitPlan implementation. (Anu Engineer via
+ Arpit Agarwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 5217740..2607800 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -869,6 +869,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_DEFAULT =
HdfsConstants.DEFAULT_DATA_SOCKET_SIZE;
+ // Disk Balancer Keys
+ public static final String DFS_DISK_BALANCER_ENABLED =
+ "dfs.disk.balancer.enabled";
+ public static final boolean DFS_DISK_BALANCER_ENABLED_DEFAULT = false;
+
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
index adb9888..782fbe7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DiskBa
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DiskBalancerSettingResponseProto;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
-import org.apache.hadoop.hdfs.server.datanode.WorkStatus;
+import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus;
/**
* Implementation for protobuf service that forwards requests
@@ -278,7 +278,7 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements
RpcController controller, QueryPlanStatusRequestProto request)
throws ServiceException {
try {
- WorkStatus result = impl.queryDiskBalancerPlan();
+ DiskBalancerWorkStatus result = impl.queryDiskBalancerPlan();
return QueryPlanStatusResponseProto
.newBuilder()
.setResult(result.getResult())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index b78f18e..69abe64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -166,7 +166,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
import org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer;
-import org.apache.hadoop.hdfs.server.diskbalancer.DiskbalancerException;
+import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -384,6 +384,8 @@ public class DataNode extends ReconfigurableBase
private static final int NUM_CORES = Runtime.getRuntime()
.availableProcessors();
private static final double CONGESTION_RATIO = 1.5;
+ private DiskBalancer diskBalancer;
+
private static Tracer createTracer(Configuration conf) {
return new Tracer.Builder("DataNode").
@@ -1010,7 +1012,33 @@ public class DataNode extends ReconfigurableBase
directoryScanner.shutdown();
}
}
-
+
+ /**
+ * Initilizes {@link DiskBalancer}.
+ * @param data - FSDataSet
+ * @param conf - Config
+ */
+ private synchronized void initDiskBalancer(FsDatasetSpi data,
+ Configuration conf) {
+ if (this.diskBalancer != null) {
+ return;
+ }
+
+ DiskBalancer.BlockMover mover = new DiskBalancer.DiskBalancerMover(data,
+ conf);
+ this.diskBalancer = new DiskBalancer(getDatanodeUuid(), conf, mover);
+ }
+
+ /**
+ * Shutdown disk balancer.
+ */
+ private synchronized void shutdownDiskBalancer() {
+ if (this.diskBalancer != null) {
+ this.diskBalancer.shutdown();
+ this.diskBalancer = null;
+ }
+ }
+
private void initDataXceiver(Configuration conf) throws IOException {
// find free port or use privileged port provided
TcpPeerServer tcpPeerServer;
@@ -1491,6 +1519,7 @@ public class DataNode extends ReconfigurableBase
data.addBlockPool(nsInfo.getBlockPoolID(), conf);
blockScanner.enableBlockPoolId(bpos.getBlockPoolId());
initDirectoryScanner(conf);
+ initDiskBalancer(data, conf);
}
List<BPOfferService> getAllBpOs() {
@@ -1814,6 +1843,7 @@ public class DataNode extends ReconfigurableBase
// Terminate directory scanner and block scanner
shutdownPeriodicScanners();
+ shutdownDiskBalancer();
// Stop the web server
if (httpServer != null) {
@@ -3224,31 +3254,30 @@ public class DataNode extends ReconfigurableBase
* @param bandwidth - Max disk bandwidth to use, 0 means use value defined
* in the configration.
* @param plan - Actual plan
- * @return success or throws an exception.
- * @throws Exception
+ * @throws IOException
*/
@Override
public void submitDiskBalancerPlan(String planID,
long planVersion, long bandwidth, String plan) throws IOException {
- // TODO : This will be replaced with actual code later.
- // Right now throwing DiskbalancerException instead
- // NotImplementedException to indicate the eventually disk balancer code
- // will throw DiskbalancerException.
- throw new DiskbalancerException("Not Implemented", 0);
+ checkSuperuserPrivilege();
+ // TODO : Support force option
+ this.diskBalancer.submitPlan(planID, planVersion, plan, bandwidth, false);
}
@Override
public void cancelDiskBalancePlan(String planID) throws
IOException {
checkSuperuserPrivilege();
- throw new DiskbalancerException("Not Implemented", 0);
+ throw new DiskBalancerException("Not Implemented",
+ DiskBalancerException.Result.INTERNAL_ERROR);
}
@Override
- public WorkStatus queryDiskBalancerPlan() throws IOException {
+ public DiskBalancerWorkStatus queryDiskBalancerPlan() throws IOException {
checkSuperuserPrivilege();
- throw new DiskbalancerException("Not Implemented", 0);
+ throw new DiskBalancerException("Not Implemented",
+ DiskBalancerException.Result.INTERNAL_ERROR);
}
/**
@@ -3262,6 +3291,7 @@ public class DataNode extends ReconfigurableBase
@Override
public String getDiskBalancerSetting(String key) throws IOException {
checkSuperuserPrivilege();
- throw new DiskbalancerException("Not Implemented", 0);
+ throw new DiskBalancerException("Not Implemented",
+ DiskBalancerException.Result.INTERNAL_ERROR);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
new file mode 100644
index 0000000..1c8ba4cf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -0,0 +1,542 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerConstants;
+import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
+import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
+import org.apache.hadoop.hdfs.server.diskbalancer.planner.Step;
+import org.apache.hadoop.util.Time;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Worker class for Disk Balancer.
+ * <p/>
+ * Here is the high level logic executed by this class. Users can submit disk
+ * balancing plans using submitPlan calls. After a set of sanity checks the plan
+ * is admitted and put into workMap.
+ * <p/>
+ * The executePlan launches a thread that picks up work from workMap and hands
+ * it over to the BlockMover#copyBlocks function.
+ * <p/>
+ * Constraints :
+ * <p/>
+ * Only one plan can be executing in a datanode at any given time. This is
+ * ensured by checking the future handle of the worker thread in submitPlan.
+ */
+@InterfaceAudience.Private
+public class DiskBalancer {
+
+ private static final Log LOG = LogFactory.getLog(DiskBalancer.class);
+ private final FsDatasetSpi<?> dataset;
+ private final String dataNodeUUID;
+ private final BlockMover blockMover;
+ private final ReentrantLock lock;
+ private final ConcurrentHashMap<VolumePair, DiskBalancerWorkItem> workMap;
+ private boolean isDiskBalancerEnabled = false;
+ private ExecutorService scheduler;
+ private Future future;
+ private String planID;
+
+ /**
+ * Constructs a Disk Balancer object. This object takes care of reading a
+ * NodePlan and executing it against a set of volumes.
+ *
+ * @param dataNodeUUID - Data node UUID
+ * @param conf - Hdfs Config
+ * @param blockMover - Object that supports moving blocks.
+ */
+ public DiskBalancer(String dataNodeUUID,
+ Configuration conf, BlockMover blockMover) {
+ this.blockMover = blockMover;
+ this.dataset = this.blockMover.getDataset();
+ this.dataNodeUUID = dataNodeUUID;
+ scheduler = Executors.newSingleThreadExecutor();
+ lock = new ReentrantLock();
+ workMap = new ConcurrentHashMap<>();
+ this.isDiskBalancerEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_DISK_BALANCER_ENABLED,
+ DFSConfigKeys.DFS_DISK_BALANCER_ENABLED_DEFAULT);
+ }
+
+ /**
+ * Shutdown disk balancer services.
+ */
+ public void shutdown() {
+ lock.lock();
+ try {
+ this.isDiskBalancerEnabled = false;
+ if ((this.future != null) && (!this.future.isDone())) {
+ this.blockMover.setExitFlag();
+ shutdownExecutor();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Shutdown the executor.
+ */
+ private void shutdownExecutor() {
+ scheduler.shutdown();
+ try {
+ if(!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+ scheduler.shutdownNow();
+ if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) {
+ LOG.error("Disk Balancer : Scheduler did not terminate.");
+ }
+ }
+ } catch (InterruptedException ex) {
+ scheduler.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Takes a client submitted plan and converts into a set of work items that
+ * can be executed by the blockMover.
+ *
+ * @param planID - A SHA512 of the plan string
+ * @param planVersion - version of the plan string - for future use.
+ * @param plan - Actual Plan
+ * @param bandwidth - BytesPerSec to copy
+ * @param force - Skip some validations and execute the plan file.
+ * @throws DiskBalancerException
+ */
+ public void submitPlan(String planID, long planVersion, String plan,
+ long bandwidth, boolean force)
+ throws DiskBalancerException {
+
+ lock.lock();
+ try {
+ checkDiskBalancerEnabled();
+ if ((this.future != null) && (!this.future.isDone())) {
+ LOG.error("Disk Balancer - Executing another plan, submitPlan failed.");
+ throw new DiskBalancerException("Executing another plan",
+ DiskBalancerException.Result.PLAN_ALREADY_IN_PROGRESS);
+ }
+ NodePlan nodePlan =
+ verifyPlan(planID, planVersion, plan, bandwidth, force);
+ createWorkPlan(nodePlan);
+ this.planID = planID;
+ executePlan();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Throws if Disk balancer is disabled.
+ *
+ * @throws DiskBalancerException
+ */
+ private void checkDiskBalancerEnabled()
+ throws DiskBalancerException {
+ if (!isDiskBalancerEnabled) {
+ LOG.error("Disk Balancer is not enabled.");
+ throw new DiskBalancerException("Disk Balancer is not enabled.",
+ DiskBalancerException.Result.DISK_BALANCER_NOT_ENABLED);
+ }
+ }
+
+ /**
+ * Verifies that user provided plan is valid.
+ *
+ * @param planID - SHA 512 of the plan.
+ * @param planVersion - Version of the plan, for future use.
+ * @param plan - Plan String in Json.
+ * @param bandwidth - Max disk bandwidth to use per second.
+ * @param force - Skip verifying when the plan was generated.
+ * @return a NodePlan Object.
+ * @throws DiskBalancerException
+ */
+ private NodePlan verifyPlan(String planID, long planVersion, String plan,
+ long bandwidth, boolean force)
+ throws DiskBalancerException {
+
+ Preconditions.checkState(lock.isHeldByCurrentThread());
+ verifyPlanVersion(planVersion);
+ NodePlan nodePlan = verifyPlanHash(planID, plan);
+ if (!force) {
+ verifyTimeStamp(nodePlan);
+ }
+ verifyNodeUUID(nodePlan);
+ return nodePlan;
+ }
+
+ /**
+ * Verifies the plan version is something that we support.
+ *
+ * @param planVersion - Long version.
+ * @throws DiskBalancerException
+ */
+ private void verifyPlanVersion(long planVersion)
+ throws DiskBalancerException {
+ if ((planVersion < DiskBalancerConstants.DISKBALANCER_MIN_VERSION) ||
+ (planVersion > DiskBalancerConstants.DISKBALANCER_MAX_VERSION)) {
+ LOG.error("Disk Balancer - Invalid plan version.");
+ throw new DiskBalancerException("Invalid plan version.",
+ DiskBalancerException.Result.INVALID_PLAN_VERSION);
+ }
+ }
+
+ /**
+ * Verifies that plan matches the SHA512 provided by the client.
+ *
+ * @param planID - Sha512 Hex Bytes
+ * @param plan - Plan String
+ * @throws DiskBalancerException
+ */
+ private NodePlan verifyPlanHash(String planID, String plan)
+ throws DiskBalancerException {
+ final long sha512Length = 128;
+ if (plan == null || plan.length() == 0) {
+ LOG.error("Disk Balancer - Invalid plan.");
+ throw new DiskBalancerException("Invalid plan.",
+ DiskBalancerException.Result.INVALID_PLAN);
+ }
+
+ if ((planID == null) ||
+ (planID.length() != sha512Length) ||
+ !DigestUtils.sha512Hex(plan.getBytes(Charset.forName("UTF-8")))
+ .equalsIgnoreCase(planID)) {
+ LOG.error("Disk Balancer - Invalid plan hash.");
+ throw new DiskBalancerException("Invalid or mis-matched hash.",
+ DiskBalancerException.Result.INVALID_PLAN_HASH);
+ }
+
+ try {
+ return NodePlan.parseJson(plan);
+ } catch (IOException ex) {
+ throw new DiskBalancerException("Parsing plan failed.", ex,
+ DiskBalancerException.Result.MALFORMED_PLAN);
+ }
+ }
+
+ /**
+ * Verifies that this plan is not older than 24 hours.
+ *
+ * @param plan - Node Plan
+ */
+ private void verifyTimeStamp(NodePlan plan) throws DiskBalancerException {
+ long now = Time.now();
+ long planTime = plan.getTimeStamp();
+
+ // TODO : Support Valid Plan hours as a user configurable option.
+ if ((planTime +
+ (TimeUnit.HOURS.toMillis(
+ DiskBalancerConstants.DISKBALANCER_VALID_PLAN_HOURS))) < now) {
+ String hourString = "Plan was generated more than " +
+ Integer.toString(DiskBalancerConstants.DISKBALANCER_VALID_PLAN_HOURS)
+ + " hours ago.";
+ LOG.error("Disk Balancer - " + hourString);
+ throw new DiskBalancerException(hourString,
+ DiskBalancerException.Result.OLD_PLAN_SUBMITTED);
+ }
+ }
+
+ /**
+ * Verify Node UUID.
+ *
+ * @param plan - Node Plan
+ */
+ private void verifyNodeUUID(NodePlan plan) throws DiskBalancerException {
+ if ((plan.getNodeUUID() == null) ||
+ !plan.getNodeUUID().equals(this.dataNodeUUID)) {
+ LOG.error("Disk Balancer - Plan was generated for another node.");
+ throw new DiskBalancerException(
+ "Plan was generated for another node.",
+ DiskBalancerException.Result.DATANODE_ID_MISMATCH);
+ }
+ }
+
+ /**
+ * Convert a node plan to DiskBalancerWorkItem that Datanode can execute.
+ *
+ * @param plan - Node Plan
+ */
+ private void createWorkPlan(NodePlan plan) throws DiskBalancerException {
+ Preconditions.checkState(lock.isHeldByCurrentThread());
+
+ // Cleanup any residual work in the map.
+ workMap.clear();
+ Map<String, FsVolumeSpi> pathMap = getStorageIDToVolumeMap();
+
+ for (Step step : plan.getVolumeSetPlans()) {
+ String sourceuuid = step.getSourceVolume().getUuid();
+ String destinationuuid = step.getDestinationVolume().getUuid();
+
+ FsVolumeSpi sourceVol = pathMap.get(sourceuuid);
+ if (sourceVol == null) {
+ LOG.error("Disk Balancer - Unable to find source volume. submitPlan " +
+ "failed.");
+ throw new DiskBalancerException("Unable to find source volume.",
+ DiskBalancerException.Result.INVALID_VOLUME);
+ }
+
+ FsVolumeSpi destVol = pathMap.get(destinationuuid);
+ if (destVol == null) {
+ LOG.error("Disk Balancer - Unable to find destination volume. " +
+ "submitPlan failed.");
+ throw new DiskBalancerException("Unable to find destination volume.",
+ DiskBalancerException.Result.INVALID_VOLUME);
+ }
+ createWorkPlan(sourceVol, destVol, step.getBytesToMove());
+ }
+ }
+
+ /**
+ * Returns a path to Volume Map.
+ *
+ * @return Map
+ * @throws DiskBalancerException
+ */
+ private Map<String, FsVolumeSpi> getStorageIDToVolumeMap()
+ throws DiskBalancerException {
+ Map<String, FsVolumeSpi> pathMap = new HashMap<>();
+ FsDatasetSpi.FsVolumeReferences references;
+ try {
+ synchronized (this.dataset) {
+ references = this.dataset.getFsVolumeReferences();
+ for (int ndx = 0; ndx < references.size(); ndx++) {
+ FsVolumeSpi vol = references.get(ndx);
+ pathMap.put(vol.getStorageID(), vol);
+ }
+ references.close();
+ }
+ } catch (IOException ex) {
+ LOG.error("Disk Balancer - Internal Error.", ex);
+ throw new DiskBalancerException("Internal error", ex,
+ DiskBalancerException.Result.INTERNAL_ERROR);
+ }
+ return pathMap;
+ }
+
+ /**
+ * Starts Executing the plan, exits when the plan is done executing.
+ */
+ private void executePlan() {
+ Preconditions.checkState(lock.isHeldByCurrentThread());
+ this.blockMover.setRunnable();
+ if (this.scheduler.isShutdown()) {
+ this.scheduler = Executors.newSingleThreadExecutor();
+ }
+
+ this.future = scheduler.submit(new Runnable() {
+ @Override
+ public void run() {
+ Thread.currentThread().setName("DiskBalancerThread");
+ LOG.info("Executing Disk balancer plan. Plan ID - " + planID);
+
+ for (Map.Entry<VolumePair, DiskBalancerWorkItem> entry :
+ workMap.entrySet()) {
+ blockMover.copyBlocks(entry.getKey(), entry.getValue());
+ }
+ }
+ });
+ }
+
+ /**
+ * Insert work items to work map.
+ *
+ * @param source - Source vol
+ * @param dest - destination volume
+ * @param bytesToMove - number of bytes to move
+ */
+ private void createWorkPlan(FsVolumeSpi source, FsVolumeSpi dest,
+ long bytesToMove) throws DiskBalancerException {
+
+ if(source.getStorageID().equals(dest.getStorageID())) {
+ throw new DiskBalancerException("Same source and destination",
+ DiskBalancerException.Result.INVALID_MOVE);
+ }
+ VolumePair pair = new VolumePair(source, dest);
+
+ // In case we have a plan with more than
+ // one line of same <source, dest>
+ // we compress that into one work order.
+ if (workMap.containsKey(pair)) {
+ bytesToMove += workMap.get(pair).getBytesToCopy();
+ }
+
+ DiskBalancerWorkItem work = new DiskBalancerWorkItem(bytesToMove, 0);
+ workMap.put(pair, work);
+ }
+
+ /**
+ * BlockMover supports moving blocks across Volumes.
+ */
+ public interface BlockMover {
+ /**
+ * Copies blocks from a set of volumes.
+ *
+ * @param pair - Source and Destination Volumes.
+ * @param item - Number of bytes to move from volumes.
+ */
+ void copyBlocks(VolumePair pair, DiskBalancerWorkItem item);
+
+ /**
+ * Begin the actual copy operations. This is useful in testing.
+ */
+ void setRunnable();
+
+ /**
+ * Tells copyBlocks to exit from the copy routine.
+ */
+ void setExitFlag();
+
+ /**
+ * Returns a pointer to the current dataset we are operating against.
+ *
+ * @return FsDatasetSpi
+ */
+ FsDatasetSpi getDataset();
+ }
+
+ /**
+ * Holds references to actual volumes that we will be operating against.
+ */
+ static class VolumePair {
+ private final FsVolumeSpi source;
+ private final FsVolumeSpi dest;
+
+ /**
+ * Constructs a volume pair.
+ *
+ * @param source - Source Volume
+ * @param dest - Destination Volume
+ */
+ public VolumePair(FsVolumeSpi source, FsVolumeSpi dest) {
+ this.source = source;
+ this.dest = dest;
+ }
+
+ /**
+ * gets source volume.
+ *
+ * @return volume
+ */
+ public FsVolumeSpi getSource() {
+ return source;
+ }
+
+ /**
+ * Gets Destination volume.
+ *
+ * @return volume.
+ */
+ public FsVolumeSpi getDest() {
+ return dest;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ VolumePair that = (VolumePair) o;
+ return source.equals(that.source) && dest.equals(that.dest);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = source.getBasePath().hashCode();
+ result = 31 * result + dest.getBasePath().hashCode();
+ return result;
+ }
+ }
+
+ /**
+ * Actual DataMover class for DiskBalancer.
+ * <p/>
+ * TODO : Add implementation for this class. This is here as a place holder so
+ * that Datanode can make calls into this class.
+ */
+ public static class DiskBalancerMover implements BlockMover {
+ private final FsDatasetSpi dataset;
+
+ /**
+ * Constructs diskBalancerMover.
+ *
+ * @param dataset Dataset
+ * @param conf Configuration
+ */
+ public DiskBalancerMover(FsDatasetSpi dataset, Configuration conf) {
+ this.dataset = dataset;
+ // TODO : Read Config values.
+ }
+
+ /**
+ * Copies blocks from a set of volumes.
+ *
+ * @param pair - Source and Destination Volumes.
+ * @param item - Number of bytes to move from volumes.
+ */
+ @Override
+ public void copyBlocks(VolumePair pair, DiskBalancerWorkItem item) {
+
+ }
+
+ /**
+ * Begin the actual copy operations. This is useful in testing.
+ */
+ @Override
+ public void setRunnable() {
+
+ }
+
+ /**
+ * Tells copyBlocks to exit from the copy routine.
+ */
+ @Override
+ public void setExitFlag() {
+
+ }
+
+ /**
+ * Returns a pointer to the current dataset we are operating against.
+ *
+ * @return FsDatasetSpi
+ */
+ @Override
+ public FsDatasetSpi getDataset() {
+ return dataset;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerConstants.java
index 553827e..7144a50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerConstants.java
@@ -29,6 +29,15 @@ public final class DiskBalancerConstants {
public static final String DISKBALANCER_VOLUME_NAME =
"DiskBalancerVolumeName";
+ /** Min and Max Plan file versions that we know of. **/
+ public static final int DISKBALANCER_MIN_VERSION = 1;
+ public static final int DISKBALANCER_MAX_VERSION = 1;
+
+ /**
+ * We treat a plan as stale if it was generated before the hours
+ * defined by the constant below. Defaults to 24 hours.
+ */
+ public static final int DISKBALANCER_VALID_PLAN_HOURS = 24;
// never constructed.
private DiskBalancerConstants() {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
new file mode 100644
index 0000000..a5e1581
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.diskbalancer;
+
+import java.io.IOException;
+
+/**
+ * Disk Balancer Exceptions.
+ */
+public class DiskBalancerException extends IOException {
+ /** Possible results from DiskBalancer. **/
+ public enum Result {
+ DISK_BALANCER_NOT_ENABLED,
+ INVALID_PLAN_VERSION,
+ INVALID_PLAN,
+ INVALID_PLAN_HASH,
+ OLD_PLAN_SUBMITTED,
+ DATANODE_ID_MISMATCH,
+ MALFORMED_PLAN,
+ PLAN_ALREADY_IN_PROGRESS,
+ INVALID_VOLUME,
+ INVALID_MOVE,
+ INTERNAL_ERROR
+ }
+
+ private final Result result;
+
+ /**
+ * Constructs an {@code IOException} with the specified detail message.
+ *
+ * @param message The detail message (which is saved for later retrieval by
+ * the {@link #getMessage()} method)
+ */
+ public DiskBalancerException(String message, Result result) {
+ super(message);
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified detail message and
+ * cause.
+ * <p/>
+ * <p> Note that the detail message associated with {@code cause} is
+ * <i>not</i>
+ * automatically incorporated into this exception's detail message.
+ *
+ * @param message The detail message (which is saved for later retrieval by
+ * the
+ * {@link #getMessage()} method)
+ * @param cause The cause (which is saved for later retrieval by the {@link
+ * #getCause()} method). (A null value is permitted, and
+ * indicates that the cause is nonexistent or unknown.)
+ */
+ public DiskBalancerException(String message, Throwable cause, Result result) {
+ super(message, cause);
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified cause and a detail
+ * message of {@code (cause==null ? null : cause.toString())} (which typically
+ * contains the class and detail message of {@code cause}). This
+ * constructor is useful for IO exceptions that are little more than
+ * wrappers for other throwables.
+ *
+ * @param cause The cause (which is saved for later retrieval by the {@link
+ * #getCause()} method). (A null value is permitted, and
+ * indicates
+ * that the cause is nonexistent or unknown.)
+ */
+ public DiskBalancerException(Throwable cause, Result result) {
+ super(cause);
+ this.result = result;
+ }
+
+ /**
+ * Returns the result.
+ * @return int
+ */
+ public Result getResult() {
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskbalancerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskbalancerException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskbalancerException.java
deleted file mode 100644
index 9d47dc3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskbalancerException.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdfs.server.diskbalancer;
-
-import java.io.IOException;
-
-/**
- * Disk Balancer Exceptions.
- */
-public class DiskbalancerException extends IOException {
- private int result;
-
- /**
- * Constructs an {@code IOException} with the specified detail message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the
- * {@link #getMessage()} method)
- */
- public DiskbalancerException(String message, int result) {
- super(message);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message and
- * cause.
- * <p/>
- * <p> Note that the detail message associated with {@code cause} is
- * <i>not</i>
- * automatically incorporated into this exception's detail message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the
- * {@link #getMessage()} method)
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and
- * indicates that the cause is nonexistent or unknown.)
- * @since 1.6
- */
- public DiskbalancerException(String message, Throwable cause, int result) {
- super(message, cause);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified cause and a detail
- * message of {@code (cause==null ? null : cause.toString())} (which typically
- * contains the class and detail message of {@code cause}). This
- * constructor is
- * useful for IO exceptions that are little more than wrappers for other
- * throwables.
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and
- * indicates
- * that the cause is nonexistent or unknown.)
- * @since 1.6
- */
- public DiskbalancerException(Throwable cause, int result) {
- super(cause);
- this.result = result;
- }
-
- /**
- * Returns the result.
- * @return int
- */
- public int getResult() {
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
index af9e9af..c86fc9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
@@ -358,4 +358,18 @@ public class DiskBalancerCluster {
return (10 - modValue) + threadRatio;
}
}
+
+ /**
+ * Returns a node by UUID.
+ * @param uuid - Node's UUID
+ * @return DiskBalancerDataNode.
+ */
+ public DiskBalancerDataNode getNodeByUUID(String uuid) {
+ for(DiskBalancerDataNode node : this.getNodes()) {
+ if(node.getDataNodeUUID().equals(uuid)) {
+ return node;
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f72fcca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
index 143b776..dc24787 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -35,9 +36,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.io.IOException;
-import java.net.URI;
-
public class TestDiskBalancerRPC {
@Rule
public ExpectedException thrown = ExpectedException.none();
@@ -48,6 +46,7 @@ public class TestDiskBalancerRPC {
@Before
public void setUp() throws Exception {
conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
}
@@ -72,22 +71,19 @@ public class TestDiskBalancerRPC {
Assert.assertEquals(cluster.getDataNodes().size(),
diskBalancerCluster.getNodes().size());
diskBalancerCluster.setNodesToProcess(diskBalancerCluster.getNodes());
- DiskBalancerDataNode node = diskBalancerCluster.getNodes().get(dnIndex);
+
+ DataNode dataNode = cluster.getDataNodes().get(dnIndex);
+ DiskBalancerDataNode node = diskBalancerCluster.getNodeByUUID(
+ dataNode.getDatanodeUuid());
GreedyPlanner planner = new GreedyPlanner(10.0f, node);
NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort
());
planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
- final int planVersion = 0; // So far we support only one version.
- DataNode dataNode = cluster.getDataNodes().get(dnIndex);
+ final int planVersion = 1; // So far we support only one version.
String planHash = DigestUtils.sha512Hex(plan.toJson());
- // Since submitDiskBalancerPlan is not implemented yet, it throws an
- // Exception, this will be modified with the actual implementation.
- thrown.expect(DiskbalancerException.class);
dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
-
-
}
@Test
@@ -117,10 +113,10 @@ public class TestDiskBalancerRPC {
// Exception, this will be modified with the actual implementation.
try {
dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
- } catch (DiskbalancerException ex) {
+ } catch (DiskBalancerException ex) {
// Let us ignore this for time being.
}
- thrown.expect(DiskbalancerException.class);
+ thrown.expect(DiskBalancerException.class);
dataNode.cancelDiskBalancePlan(planHash);
}
@@ -152,13 +148,13 @@ public class TestDiskBalancerRPC {
// Exception, this will be modified with the actual implementation.
try {
dataNode.submitDiskBalancerPlan(planHash, planVersion, 10, plan.toJson());
- } catch (DiskbalancerException ex) {
+ } catch (DiskBalancerException ex) {
// Let us ignore this for time being.
}
// TODO : This will be fixed when we have implementation for this
// function in server side.
- thrown.expect(DiskbalancerException.class);
+ thrown.expect(DiskBalancerException.class);
dataNode.queryDiskBalancerPlan();
}
@@ -166,7 +162,7 @@ public class TestDiskBalancerRPC {
public void testgetDiskBalancerSetting() throws Exception {
final int dnIndex = 0;
DataNode dataNode = cluster.getDataNodes().get(dnIndex);
- thrown.expect(DiskbalancerException.class);
+ thrown.expect(DiskBalancerException.class);
dataNode.getDiskBalancerSetting(
DiskBalancerConstants.DISKBALANCER_BANDWIDTH);
}
[40/50] [abbrv] hadoop git commit: MAPREDUCE-6613. Change
mapreduce.jobhistory.jhist.format default from json to binary (rchiang via
rkanter)
Posted by ar...@apache.org.
MAPREDUCE-6613. Change mapreduce.jobhistory.jhist.format default from json to binary (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6eae4337
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6eae4337
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6eae4337
Branch: refs/heads/HDFS-1312
Commit: 6eae4337d1929077ffa74734327775fb987ba910
Parents: 3757785
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Feb 19 17:15:54 2016 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Feb 19 17:15:54 2016 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java | 2 +-
.../src/main/resources/mapred-default.xml | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eae4337/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index bf9d824..f0ad171 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -16,6 +16,9 @@ Trunk (Unreleased)
MAPREDUCE-6336. Enable v2 FileOutputCommitter by default.
(Siqi Li via gera)
+ MAPREDUCE-6613. Change mapreduce.jobhistory.jhist.format default from
+ json to binary (rchiang via rkanter)
+
NEW FEATURES
MAPREDUCE-778. Rumen Anonymizer. (Amar Kamat and Chris Douglas via amarrk)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eae4337/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 86dfad3..2277ef6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -227,5 +227,5 @@ public class JHAdminConfig {
public static final String MR_HS_JHIST_FORMAT =
MR_HISTORY_PREFIX + "jhist.format";
public static final String DEFAULT_MR_HS_JHIST_FORMAT =
- "json";
+ "binary";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eae4337/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index efbbf53..d87085f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1754,7 +1754,7 @@
values are "json" for text output and "binary" for faster parsing.
</description>
<name>mapreduce.jobhistory.jhist.format</name>
- <value>json</value>
+ <value>binary</value>
</property>
<property>
[43/50] [abbrv] hadoop git commit: YARN-4708. Missing default mapper
type in TimelineServer performance test tool usage. Contributed by Kai
Sasaki.
Posted by ar...@apache.org.
YARN-4708. Missing default mapper type in TimelineServer performance test tool usage. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b68901d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b68901d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b68901d7
Branch: refs/heads/HDFS-1312
Commit: b68901d7dde9cb48545fcf0b94f2ac266b909a5d
Parents: d5abd29
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Mon Feb 22 07:49:50 2016 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Mon Feb 22 07:54:53 2016 +0900
----------------------------------------------------------------------
.../hadoop/mapreduce/TimelineServicePerformance.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 ++
.../src/site/markdown/TimelineServer.md | 13 +++++++------
3 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68901d7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
index 9f76659..0753d7f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
@@ -48,7 +48,7 @@ public class TimelineServicePerformance extends Configured implements Tool {
")\n" +
" [-v] timeline service version\n" +
" [-mtype <mapper type in integer>]\n" +
- " 1. simple entity write mapper\n" +
+ " 1. simple entity write mapper (default)\n" +
" 2. jobhistory files replay mapper\n" +
" [-s <(KBs)test>] number of KB per put (mtype=1, default: " +
SimpleEntityWriterV1.KBS_SENT_DEFAULT + " KB)\n" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68901d7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9b64204..d6aa4e4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -135,6 +135,8 @@ Release 2.9.0 - UNRELEASED
YARN-4689. FairScheduler: Cleanup preemptContainer to be more readable.
(Kai Sasaki via kasha)
+ YARN-4708. Missing default mapper type in TimelineServer performance test tool
+ usage. (Kai Sasaki via ozawa)
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68901d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 3e589d3..b10c860 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -783,8 +783,8 @@ String.
| `entitytype` | string | The entity type |
| `relatedentities` | map | The related entities' identifiers, which are organized in a map of entityType : [entity1, entity2, ...] |
| `events` | list | The events of the entity |
-| `primaryfilters` | map | The primary filters of the entity, which are orgainzied in a map of key : [value1, value2, ...] |
-| `otherinfo` | map | The other information of the entity, which is orgainzied in a map of key : value |
+| `primaryfilters` | map | The primary filters of the entity, which are organized in a map of key : [value1, value2, ...] |
+| `otherinfo` | map | The other information of the entity, which is organized in a map of key : value |
| `starttime` | long | The start time of the entity |
### Response Examples:
@@ -2038,7 +2038,7 @@ this failure *will not* result in an HTTP error code being retured.
A status code of 200 will be returned —however, there will be an error code
in the list of failed entities for each entity which could not be added.
-<a name="TIMELINE_SERVER_PERFORMANCE_TEST_TOOL"></a> Timelnine Server Performance Test Tool
+<a name="TIMELINE_SERVER_PERFORMANCE_TEST_TOOL"></a> Timeline Server Performance Test Tool
----------
###<a name="HIGHLIGHTS"></a>Highlights
@@ -2054,7 +2054,8 @@ Mapper Types Description:
1. SimpleEntityWriter mapper
Each mapper writes a user-specified number of timeline entities
- with a user-specified size to the timeline server.
+ with a user-specified size to the timeline server. SimpleEntityWrite
+ is a default mapper of the performance test tool.
2. JobHistoryFileReplay mapper
Each mapper replays jobhistory files under a specified directory
@@ -2071,7 +2072,7 @@ Options:
[-m <maps>] number of mappers (default: 1)
[-v] timeline service version
[-mtype <mapper type in integer>]
- 1. simple entity write mapper
+ 1. simple entity write mapper (default)
2. jobhistory files replay mapper
[-s <(KBs)test>] number of KB per put (mtype=1, default: 1 KB)
[-t] package sending iterations per mapper (mtype=1, default: 100)
@@ -2105,7 +2106,7 @@ Example input of JobHistoryFileReplay mapper test:
/testInput/job_2.jhist
/testInput/job_2_conf.xml
-Eample output of JobHistoryFileReplay test:
+Example output of JobHistoryFileReplay test:
TRANSACTION RATE (per mapper): 4000.0 ops/s
IO RATE (per mapper): 0.0 KB/s
[17/50] [abbrv] hadoop git commit: HDFS-9799. Reimplement
getCurrentTrashDir to remove incompatibility. (zhz)
Posted by ar...@apache.org.
HDFS-9799. Reimplement getCurrentTrashDir to remove incompatibility. (zhz)
Change-Id: I7834bcebffed38cb384db5395ddb8b6dd9e79a0b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fb14aac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fb14aac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fb14aac
Branch: refs/heads/HDFS-1312
Commit: 0fb14aaced7afdbbb72e59316c186ebf6ec2d091
Parents: 76fab26
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Feb 17 13:30:50 2016 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Feb 17 13:30:50 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 41 ++++++-------
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +-
.../java/org/apache/hadoop/fs/TrashPolicy.java | 2 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 9 +--
.../hadoop/hdfs/DistributedFileSystem.java | 63 +++++++++++---------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
6 files changed, 64 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fb14aac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index f4a2e7d..3e26f68 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2689,9 +2689,8 @@ public abstract class FileSystem extends Configured implements Closeable {
*
* @param path the trash root of the path to be determined.
* @return the default implementation returns "/user/$USER/.Trash".
- * @throws IOException
*/
- public Path getTrashRoot(Path path) throws IOException {
+ public Path getTrashRoot(Path path) {
return this.makeQualified(new Path(getHomeDirectory().toUri().getPath(),
TRASH_PREFIX));
}
@@ -2703,29 +2702,31 @@ public abstract class FileSystem extends Configured implements Closeable {
* @return all the trash root directories.
* Default FileSystem returns .Trash under users' home directories if
* /user/$USER/.Trash exists.
- * @throws IOException
*/
- public Collection<FileStatus> getTrashRoots(boolean allUsers)
- throws IOException {
+ public Collection<FileStatus> getTrashRoots(boolean allUsers) {
Path userHome = new Path(getHomeDirectory().toUri().getPath());
- List<FileStatus> ret = new ArrayList<FileStatus>();
- if (!allUsers) {
- Path userTrash = new Path(userHome, TRASH_PREFIX);
- if (exists(userTrash)) {
- ret.add(getFileStatus(userTrash));
- }
- } else {
- Path homeParent = userHome.getParent();
- if (exists(homeParent)) {
- FileStatus[] candidates = listStatus(homeParent);
- for (FileStatus candidate : candidates) {
- Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX);
- if (exists(userTrash)) {
- candidate.setPath(userTrash);
- ret.add(candidate);
+ List<FileStatus> ret = new ArrayList<>();
+ try {
+ if (!allUsers) {
+ Path userTrash = new Path(userHome, TRASH_PREFIX);
+ if (exists(userTrash)) {
+ ret.add(getFileStatus(userTrash));
+ }
+ } else {
+ Path homeParent = userHome.getParent();
+ if (exists(homeParent)) {
+ FileStatus[] candidates = listStatus(homeParent);
+ for (FileStatus candidate : candidates) {
+ Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX);
+ if (exists(userTrash)) {
+ candidate.setPath(userTrash);
+ ret.add(candidate);
+ }
}
}
}
+ } catch (IOException e) {
+ LOG.warn("Cannot get all trash roots", e);
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fb14aac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 53678e0..4ee7514 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -645,13 +645,12 @@ public class FilterFileSystem extends FileSystem {
}
@Override
- public Path getTrashRoot(Path path) throws IOException {
+ public Path getTrashRoot(Path path) {
return fs.getTrashRoot(path);
}
@Override
- public Collection<FileStatus> getTrashRoots(boolean allUsers)
- throws IOException {
+ public Collection<FileStatus> getTrashRoots(boolean allUsers) {
return fs.getTrashRoots(allUsers);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fb14aac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
index 1d901c1..92a4d1f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
@@ -89,7 +89,7 @@ public abstract class TrashPolicy extends Configured {
* It returns the trash location correctly for the path specified no matter
* the path is in encryption zone or not.
*/
- public abstract Path getCurrentTrashDir() throws IOException;
+ public abstract Path getCurrentTrashDir();
/**
* Get the current trash directory for path specified based on the Trash
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fb14aac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index b5d245b..012ce32 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -196,7 +196,7 @@ public class TrashPolicyDefault extends TrashPolicy {
}
@Override
- public Path getCurrentTrashDir() throws IOException {
+ public Path getCurrentTrashDir() {
return new Path(fs.getTrashRoot(null), CURRENT);
}
@@ -250,12 +250,7 @@ public class TrashPolicyDefault extends TrashPolicy {
now = Time.now();
if (now >= end) {
Collection<FileStatus> trashRoots;
- try {
- trashRoots = fs.getTrashRoots(true); // list all home dirs
- } catch (IOException e) {
- LOG.warn("Trash can't list all trash roots: "+e+" Sleeping.");
- continue;
- }
+ trashRoots = fs.getTrashRoots(true); // list all trash dirs
for (FileStatus trashRoot : trashRoots) { // dump each trash
if (!trashRoot.isDirectory())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fb14aac/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 6de7659..69f3871 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2338,27 +2338,31 @@ public class DistributedFileSystem extends FileSystem {
/**
* Get the root directory of Trash for a path in HDFS.
* 1. File in encryption zone returns /ez1/.Trash/username
- * 2. File not in encryption zone returns /users/username/.Trash
+ * 2. File not in encryption zone, or encountered exception when checking
+ * the encryption zone of the path, returns /users/username/.Trash
* Caller appends either Current or checkpoint timestamp for trash destination
* @param path the trash root of the path to be determined.
* @return trash root
- * @throws IOException
*/
@Override
- public Path getTrashRoot(Path path) throws IOException {
+ public Path getTrashRoot(Path path) {
if ((path == null) || path.isRoot() || !dfs.isHDFSEncryptionEnabled()) {
return super.getTrashRoot(path);
}
String parentSrc = path.getParent().toUri().getPath();
- EncryptionZone ez = dfs.getEZForPath(parentSrc);
- if ((ez != null)) {
- return this.makeQualified(
- new Path(ez.getPath() + "/" + FileSystem.TRASH_PREFIX +
- dfs.ugi.getShortUserName()));
- } else {
- return super.getTrashRoot(path);
+ try {
+ EncryptionZone ez = dfs.getEZForPath(parentSrc);
+ if ((ez != null)) {
+ return this.makeQualified(
+ new Path(ez.getPath() + "/" + FileSystem.TRASH_PREFIX +
+ dfs.ugi.getShortUserName()));
+ }
+ } catch (IOException e) {
+ DFSClient.LOG.warn("Exception in checking the encryption zone for the " +
+ "path " + parentSrc + ". " + e.getMessage());
}
+ return super.getTrashRoot(path);
}
/**
@@ -2366,32 +2370,37 @@ public class DistributedFileSystem extends FileSystem {
* 1. File deleted from non-encryption zone /user/username/.Trash
* 2. File deleted from encryption zones
* e.g., ez1 rooted at /ez1 has its trash root at /ez1/.Trash/$USER
- * @allUsers return trashRoots of all users if true, used by emptier
+ * @param allUsers return trashRoots of all users if true, used by emptier
* @return trash roots of HDFS
- * @throws IOException
*/
@Override
- public Collection<FileStatus> getTrashRoots(boolean allUsers) throws IOException {
- List<FileStatus> ret = new ArrayList<FileStatus>();
+ public Collection<FileStatus> getTrashRoots(boolean allUsers) {
+ List<FileStatus> ret = new ArrayList<>();
// Get normal trash roots
ret.addAll(super.getTrashRoots(allUsers));
- // Get EZ Trash roots
- final RemoteIterator<EncryptionZone> it = dfs.listEncryptionZones();
- while (it.hasNext()) {
- Path ezTrashRoot = new Path(it.next().getPath(), FileSystem.TRASH_PREFIX);
- if (allUsers) {
- for (FileStatus candidate : listStatus(ezTrashRoot)) {
- if (exists(candidate.getPath())) {
- ret.add(candidate);
+ try {
+ // Get EZ Trash roots
+ final RemoteIterator<EncryptionZone> it = dfs.listEncryptionZones();
+ while (it.hasNext()) {
+ Path ezTrashRoot = new Path(it.next().getPath(),
+ FileSystem.TRASH_PREFIX);
+ if (allUsers) {
+ for (FileStatus candidate : listStatus(ezTrashRoot)) {
+ if (exists(candidate.getPath())) {
+ ret.add(candidate);
+ }
+ }
+ } else {
+ Path userTrash = new Path(ezTrashRoot, System.getProperty(
+ "user.name"));
+ if (exists(userTrash)) {
+ ret.add(getFileStatus(userTrash));
}
- }
- } else {
- Path userTrash = new Path(ezTrashRoot, System.getProperty("user.name"));
- if (exists(userTrash)) {
- ret.add(getFileStatus(userTrash));
}
}
+ } catch (IOException e){
+ DFSClient.LOG.warn("Cannot get all encrypted trash roots", e);
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fb14aac/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 36bb60d..89fc187 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2780,6 +2780,8 @@ Release 2.8.0 - UNRELEASED
HDFS-9815. Move o.a.h.fs.Hdfs to hadoop-hdfs-client.
(Vinayakumar B via wheat9)
+ HDFS-9799. Reimplement getCurrentTrashDir to remove incompatibility. (zhz)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
[48/50] [abbrv] hadoop git commit: HDFS-9842.
dfs.datanode.balance.bandwidthPerSec should accept friendly size units.
(Contributed by Lin Yiqun)
Posted by ar...@apache.org.
HDFS-9842. dfs.datanode.balance.bandwidthPerSec should accept friendly size units. (Contributed by Lin Yiqun)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a63301a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a63301a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a63301a8
Branch: refs/heads/HDFS-1312
Commit: a63301a8645ca9ab36b782d03af96e9c4e901db5
Parents: 3fab885
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Feb 22 11:34:09 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Feb 22 11:34:09 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/datanode/DataXceiverServer.java | 2 +-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 6 +++++-
3 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a63301a8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3552c8e..c39bcee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2810,6 +2810,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7452. skip StandbyException log for getCorruptFiles()
(Bramha Reddy Battula via vinayakumarb)
+ HDFS-9842. dfs.datanode.balance.bandwidthPerSec should accept friendly
+ size units. (Lin Yiqun via Arpit Agarwal)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a63301a8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
index 36cf8a1..8152e6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
@@ -131,7 +131,7 @@ class DataXceiverServer implements Runnable {
//set up parameter for cluster balancing
this.balanceThrottler = new BlockBalanceThrottler(
- conf.getLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY,
+ conf.getLongBytes(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY,
DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT),
conf.getInt(DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a63301a8/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 1560335..716cbd4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -815,7 +815,11 @@
<description>
Specifies the maximum amount of bandwidth that each datanode
can utilize for the balancing purpose in term of
- the number of bytes per second.
+ the number of bytes per second. You can use the following
+ suffix (case insensitive):
+ k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa)to specify the size
+ (such as 128k, 512m, 1g, etc.).
+ Or provide complete size in bytes (such as 134217728 for 128 MB).
</description>
</property>
[37/50] [abbrv] hadoop git commit: HDFS-9829. Erasure Coding: Improve
few exception handling logic of ErasureCodingWorker. Contributed by Rakesh R.
Posted by ar...@apache.org.
HDFS-9829. Erasure Coding: Improve few exception handling logic of ErasureCodingWorker. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6546d9e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6546d9e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6546d9e7
Branch: refs/heads/HDFS-1312
Commit: 6546d9e7ff73d2c81a803f2c61a1376a8c426987
Parents: ba1c9d4
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Feb 19 10:40:11 2016 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Feb 19 10:40:28 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../erasurecode/ErasureCodingWorker.java | 37 ++++++++------------
2 files changed, 17 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6546d9e7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1f5d65d..a377243 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -225,6 +225,9 @@ Trunk (Unreleased)
HDFS-9795. OIV Delimited should show which files are ACL-enabled (lei)
+ HDFS-9829. Erasure Coding: Improve few exception handling logic of
+ ErasureCodingWorker. (Rakesh R via jing9)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6546d9e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 60c8417..b08aa2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -124,7 +124,7 @@ public final class ErasureCodingWorker {
}
private void initializeStripedReadThreadPool(int num) {
- LOG.debug("Using striped reads; pool threads=" + num);
+ LOG.debug("Using striped reads; pool threads={}", num);
EC_RECONSTRUCTION_STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num,
60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
@@ -150,7 +150,7 @@ public final class ErasureCodingWorker {
}
private void initializeStripedBlkReconstructionThreadPool(int num) {
- LOG.debug("Using striped block reconstruction; pool threads=" + num);
+ LOG.debug("Using striped block reconstruction; pool threads={}" + num);
EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL = new ThreadPoolExecutor(2, num,
60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
new Daemon.DaemonFactory() {
@@ -180,8 +180,8 @@ public final class ErasureCodingWorker {
EC_RECONSTRUCTION_STRIPED_BLK_THREAD_POOL
.submit(new ReconstructAndTransferBlock(reconstructionInfo));
} catch (Throwable e) {
- LOG.warn("Failed to reconstruct striped block "
- + reconstructionInfo.getExtendedBlock().getLocalBlock(), e);
+ LOG.warn("Failed to reconstruct striped block {}",
+ reconstructionInfo.getExtendedBlock().getLocalBlock(), e);
}
}
}
@@ -476,12 +476,12 @@ public final class ErasureCodingWorker {
// Currently we don't check the acks for packets, this is similar as
// block replication.
} catch (Throwable e) {
- LOG.warn("Failed to reconstruct striped block: " + blockGroup, e);
+ LOG.warn("Failed to reconstruct striped block: {}", blockGroup, e);
} finally {
datanode.decrementXmitsInProgress();
// close block readers
for (StripedReader stripedReader : stripedReaders) {
- closeBlockReader(stripedReader.blockReader);
+ IOUtils.closeStream(stripedReader.blockReader);
}
for (int i = 0; i < targets.length; i++) {
IOUtils.closeStream(targetOutputStreams[i]);
@@ -588,7 +588,7 @@ public final class ErasureCodingWorker {
// If read failed for some source DN, we should not use it anymore
// and schedule read from another source DN.
StripedReader failedReader = stripedReaders.get(result.index);
- closeBlockReader(failedReader.blockReader);
+ IOUtils.closeStream(failedReader.blockReader);
failedReader.blockReader = null;
resultIndex = scheduleNewRead(used, reconstructLength,
corruptionMap);
@@ -609,6 +609,8 @@ public final class ErasureCodingWorker {
}
} catch (InterruptedException e) {
LOG.info("Read data interrupted.", e);
+ cancelReads(futures.keySet());
+ futures.clear();
break;
}
}
@@ -736,7 +738,7 @@ public final class ErasureCodingWorker {
StripedReader r = stripedReaders.get(i);
toRead = getReadLength(liveIndices[i], reconstructLen);
if (toRead > 0) {
- closeBlockReader(r.blockReader);
+ IOUtils.closeStream(r.blockReader);
r.blockReader = newBlockReader(
getBlock(blockGroup, liveIndices[i]), positionInBlock,
sources[i]);
@@ -784,8 +786,8 @@ public final class ErasureCodingWorker {
actualReadFromBlock(reader.blockReader, buf);
return null;
} catch (ChecksumException e) {
- LOG.warn("Found Checksum error for " + reader.block + " from "
- + reader.source + " at " + e.getPos());
+ LOG.warn("Found Checksum error for {} from {} at {}", reader.block,
+ reader.source, e.getPos());
addCorruptedBlock(reader.block, reader.source, corruptionMap);
throw e;
} catch (IOException e) {
@@ -837,17 +839,6 @@ public final class ErasureCodingWorker {
}
}
- // close block reader
- private void closeBlockReader(BlockReader blockReader) {
- try {
- if (blockReader != null) {
- blockReader.close();
- }
- } catch (IOException e) {
- // ignore
- }
- }
-
private InetSocketAddress getSocketAddress4Transfer(DatanodeInfo dnInfo) {
return NetUtils.createSocketAddr(dnInfo.getXferAddr(
datanode.getDnConf().getConnectToDnViaHostname()));
@@ -867,8 +858,6 @@ public final class ErasureCodingWorker {
* read directly from DN and need to check the replica is FINALIZED
* state, notice we should not use short-circuit local read which
* requires config for domain-socket in UNIX or legacy config in Windows.
- *
- * TODO: add proper tracer
*/
return RemoteBlockReader2.newBlockReader(
"dummy", block, blockToken, offsetInBlock,
@@ -876,6 +865,8 @@ public final class ErasureCodingWorker {
"", newConnectedPeer(block, dnAddr, blockToken, dnInfo), dnInfo,
null, cachingStrategy, datanode.getTracer());
} catch (IOException e) {
+ LOG.debug("Exception while creating remote block reader, datanode {}",
+ dnInfo, e);
return null;
}
}
[28/50] [abbrv] hadoop git commit: HADOOP-10865. Add a Crc32 chunked
verification benchmark for both directly and non-directly buffer cases.
Posted by ar...@apache.org.
HADOOP-10865. Add a Crc32 chunked verification benchmark for both directly and non-directly buffer cases.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbfaf3c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbfaf3c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbfaf3c2
Branch: refs/heads/HDFS-1312
Commit: bbfaf3c2712c9ba82b0f8423bdeb314bf505a692
Parents: ba6d5ed
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Feb 18 11:47:33 2016 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Feb 18 11:47:33 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/util/DataChecksum.java | 147 ++++----
.../hadoop/util/Crc32PerformanceTest.java | 350 +++++++++++++++++++
.../apache/hadoop/util/TestDataChecksum.java | 6 +
.../hadoop/hdfs/TestDFSClientRetries.java | 8 +-
5 files changed, 450 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfaf3c2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 41ba87d..6cdcb7a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1175,6 +1175,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12758. Extend CSRF Filter with UserAgent Checks
(Larry McCay via cnauroth)
+ HADOOP-10865. Add a Crc32 chunked verification benchmark for both directly
+ and non-directly buffer cases. (szetszwo)
+
BUG FIXES
HADOOP-12352. Delay in checkpointing Trash can leave trash for 2 intervals
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfaf3c2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index d9dc7af..faac587 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -286,94 +286,119 @@ public class DataChecksum implements Checksum {
* @throws ChecksumException if the checksums do not match
*/
public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums,
- String fileName, long basePos)
- throws ChecksumException {
+ String fileName, long basePos) throws ChecksumException {
if (type.size == 0) return;
if (data.hasArray() && checksums.hasArray()) {
- verifyChunkedSums(
- data.array(), data.arrayOffset() + data.position(), data.remaining(),
- checksums.array(), checksums.arrayOffset() + checksums.position(),
- fileName, basePos);
+ final int dataOffset = data.arrayOffset() + data.position();
+ final int crcsOffset = checksums.arrayOffset() + checksums.position();
+ verifyChunked(type, summer, data.array(), dataOffset, data.remaining(),
+ bytesPerChecksum, checksums.array(), crcsOffset, fileName, basePos);
return;
}
if (NativeCrc32.isAvailable()) {
NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
fileName, basePos);
- return;
+ } else {
+ verifyChunked(type, summer, data, bytesPerChecksum, checksums, fileName,
+ basePos);
}
-
- int startDataPos = data.position();
+ }
+
+ static void verifyChunked(final Type type, final Checksum algorithm,
+ final ByteBuffer data, final int bytesPerCrc, final ByteBuffer crcs,
+ final String filename, final long basePos) throws ChecksumException {
+ final byte[] bytes = new byte[bytesPerCrc];
+ final int dataOffset = data.position();
+ final int dataLength = data.remaining();
data.mark();
- checksums.mark();
+ crcs.mark();
+
try {
- byte[] buf = new byte[bytesPerChecksum];
- byte[] sum = new byte[type.size];
- while (data.remaining() > 0) {
- int n = Math.min(data.remaining(), bytesPerChecksum);
- checksums.get(sum);
- data.get(buf, 0, n);
- summer.reset();
- summer.update(buf, 0, n);
- int calculated = (int)summer.getValue();
- int stored = (sum[0] << 24 & 0xff000000) |
- (sum[1] << 16 & 0xff0000) |
- (sum[2] << 8 & 0xff00) |
- sum[3] & 0xff;
- if (calculated != stored) {
- long errPos = basePos + data.position() - startDataPos - n;
- throw new ChecksumException(
- "Checksum error: "+ fileName + " at "+ errPos +
- " exp: " + stored + " got: " + calculated, errPos);
+ int i = 0;
+ for(final int n = dataLength - bytesPerCrc + 1; i < n; i += bytesPerCrc) {
+ data.get(bytes);
+ algorithm.reset();
+ algorithm.update(bytes, 0, bytesPerCrc);
+ final int computed = (int)algorithm.getValue();
+ final int expected = crcs.getInt();
+
+ if (computed != expected) {
+ long errPos = basePos + data.position() - dataOffset - bytesPerCrc;
+ throwChecksumException(type, algorithm, filename, errPos, expected,
+ computed);
+ }
+ }
+
+ final int remainder = dataLength - i;
+ if (remainder > 0) {
+ data.get(bytes, 0, remainder);
+ algorithm.reset();
+ algorithm.update(bytes, 0, remainder);
+ final int computed = (int)algorithm.getValue();
+ final int expected = crcs.getInt();
+
+ if (computed != expected) {
+ long errPos = basePos + data.position() - dataOffset - remainder;
+ throwChecksumException(type, algorithm, filename, errPos, expected,
+ computed);
}
}
} finally {
data.reset();
- checksums.reset();
+ crcs.reset();
}
}
-
+
/**
* Implementation of chunked verification specifically on byte arrays. This
* is to avoid the copy when dealing with ByteBuffers that have array backing.
*/
- private void verifyChunkedSums(
- byte[] data, int dataOff, int dataLen,
- byte[] checksums, int checksumsOff, String fileName,
- long basePos) throws ChecksumException {
- if (type.size == 0) return;
+ static void verifyChunked(final Type type, final Checksum algorithm,
+ final byte[] data, final int dataOffset, final int dataLength,
+ final int bytesPerCrc, final byte[] crcs, final int crcsOffset,
+ final String filename, final long basePos) throws ChecksumException {
+ final int dataEnd = dataOffset + dataLength;
+ int i = dataOffset;
+ int j = crcsOffset;
+ for(final int n = dataEnd-bytesPerCrc+1; i < n; i += bytesPerCrc, j += 4) {
+ algorithm.reset();
+ algorithm.update(data, i, bytesPerCrc);
+ final int computed = (int)algorithm.getValue();
+ final int expected = ((crcs[j] << 24) + ((crcs[j + 1] << 24) >>> 8))
+ + (((crcs[j + 2] << 24) >>> 16) + ((crcs[j + 3] << 24) >>> 24));
- if (NativeCrc32.isAvailable()) {
- NativeCrc32.verifyChunkedSumsByteArray(bytesPerChecksum, type.id,
- checksums, checksumsOff, data, dataOff, dataLen, fileName, basePos);
- return;
+ if (computed != expected) {
+ final long errPos = basePos + i - dataOffset;
+ throwChecksumException(type, algorithm, filename, errPos, expected,
+ computed);
+ }
}
-
- int remaining = dataLen;
- int dataPos = 0;
- while (remaining > 0) {
- int n = Math.min(remaining, bytesPerChecksum);
-
- summer.reset();
- summer.update(data, dataOff + dataPos, n);
- dataPos += n;
- remaining -= n;
-
- int calculated = (int)summer.getValue();
- int stored = (checksums[checksumsOff] << 24 & 0xff000000) |
- (checksums[checksumsOff + 1] << 16 & 0xff0000) |
- (checksums[checksumsOff + 2] << 8 & 0xff00) |
- checksums[checksumsOff + 3] & 0xff;
- checksumsOff += 4;
- if (calculated != stored) {
- long errPos = basePos + dataPos - n;
- throw new ChecksumException(
- "Checksum error: "+ fileName + " at "+ errPos +
- " exp: " + stored + " got: " + calculated, errPos);
+ final int remainder = dataEnd - i;
+ if (remainder > 0) {
+ algorithm.reset();
+ algorithm.update(data, i, remainder);
+ final int computed = (int)algorithm.getValue();
+ final int expected = ((crcs[j] << 24) + ((crcs[j + 1] << 24) >>> 8))
+ + (((crcs[j + 2] << 24) >>> 16) + ((crcs[j + 3] << 24) >>> 24));
+
+ if (computed != expected) {
+ final long errPos = basePos + i - dataOffset;
+ throwChecksumException(type, algorithm, filename, errPos, expected,
+ computed);
}
}
}
+ private static void throwChecksumException(Type type, Checksum algorithm,
+ String filename, long errPos, int expected, int computed)
+ throws ChecksumException {
+ throw new ChecksumException("Checksum " + type
+ + " not matched for file " + filename + " at position "+ errPos
+ + String.format(": expected=%X but computed=%X", expected, computed)
+ + ", algorithm=" + algorithm.getClass().getSimpleName(), errPos);
+ }
+
/**
* Calculate checksums for the given data.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfaf3c2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
new file mode 100644
index 0000000..d8963df
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java
@@ -0,0 +1,350 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.PrintStream;
+import java.lang.reflect.Constructor;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.fs.ChecksumException;
+import org.apache.log4j.Level;
+
+/**
+ * Performance tests to compare performance of Crc32 implementations
+ * This can be run from the command line with:
+ *
+ * java -cp path/to/test/classes:path/to/common/classes \
+ * 'org.apache.hadoop.util.Crc32PerformanceTest'
+ *
+ * or
+ *
+ * hadoop org.apache.hadoop.util.Crc32PerformanceTest
+ *
+ * The output is in JIRA table format.
+ */
+public class Crc32PerformanceTest {
+ static final int MB = 1024 * 1024;
+
+ static interface Crc32 {
+
+ public void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
+ String filename, long basePos) throws ChecksumException;
+
+ static final class Native implements Crc32 {
+ @Override
+ public void verifyChunked(ByteBuffer data, int bytesPerSum,
+ ByteBuffer sums, String fileName, long basePos)
+ throws ChecksumException {
+ NativeCrc32.verifyChunkedSums(bytesPerSum, DataChecksum.Type.CRC32.id,
+ sums, data, fileName, basePos);
+ }
+ }
+
+
+ static abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
+ abstract T newAlgorithm();
+
+ @Override
+ public void verifyChunked(ByteBuffer data, int bytesPerCrc,
+ ByteBuffer crcs, String filename, long basePos)
+ throws ChecksumException {
+ final Checksum algorithm = newAlgorithm();
+ if (data.hasArray() && crcs.hasArray()) {
+ DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
+ data.array(), data.position(), data.remaining(), bytesPerCrc,
+ crcs.array(), crcs.position(), filename, basePos);
+ } else {
+ DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
+ data, bytesPerCrc, crcs, filename, basePos);
+ }
+ }
+ }
+
+ static final class Zip extends AbstractCrc32<CRC32> {
+ @Override
+ public CRC32 newAlgorithm() {
+ return new CRC32();
+ }
+ }
+
+ static final class PureJava extends AbstractCrc32<PureJavaCrc32> {
+ @Override
+ public PureJavaCrc32 newAlgorithm() {
+ return new PureJavaCrc32();
+ }
+ }
+ }
+
+ final int dataLengthMB;
+ final int trials;
+ final boolean direct;
+
+ final PrintStream out = System.out;
+
+ final List<Class<? extends Crc32>> crcs = new ArrayList<>();
+
+ Crc32PerformanceTest(final int dataLengthMB, final int trials,
+ final boolean direct) {
+ this.dataLengthMB = dataLengthMB;
+ this.trials = trials;
+ this.direct = direct;
+
+ crcs.add(Crc32.Zip.class);
+ crcs.add(Crc32.PureJava.class);
+
+ if (direct && NativeCrc32.isAvailable()) {
+ crcs.add(Crc32.Native.class);
+ ((Log4JLogger)LogFactory.getLog(NativeCodeLoader.class))
+ .getLogger().setLevel(Level.ALL);
+ }
+ }
+
+ void run() throws Exception {
+ final long startTime = System.nanoTime();
+ printSystemProperties(out);
+ out.println("Data Length = " + dataLengthMB + " MB");
+ out.println("Trials = " + trials);
+ doBench(crcs);
+ out.printf("Elapsed %.1fs\n", secondsElapsed(startTime));
+ }
+
+ public static void main(String args[]) throws Exception {
+ new Crc32PerformanceTest(64, 5, true).run();
+ }
+
+ private static void printCell(String s, int width, PrintStream out) {
+ final int w = s.length() > width? s.length(): width;
+ out.printf(" %" + w + "s |", s);
+ }
+
+ private ByteBuffer allocateByteBuffer(int length) {
+ return direct? ByteBuffer.allocateDirect(length)
+ : ByteBuffer.allocate(length);
+ }
+
+ private ByteBuffer newData() {
+ final byte[] bytes = new byte[dataLengthMB << 20];
+ new Random().nextBytes(bytes);
+ final ByteBuffer dataBufs = allocateByteBuffer(bytes.length);
+ dataBufs.mark();
+ dataBufs.put(bytes);
+ dataBufs.reset();
+ return dataBufs;
+ }
+
+ private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc) {
+ final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
+ final ByteBuffer crcBufs = allocateByteBuffer(size);
+ final DataChecksum checksum = DataChecksum.newDataChecksum(
+ DataChecksum.Type.CRC32, bytePerCrc);
+ checksum.calculateChunkedSums(dataBufs, crcBufs);
+ return crcBufs;
+ }
+
+ private void doBench(final List<Class<? extends Crc32>> crcs)
+ throws Exception {
+ final ByteBuffer[] dataBufs = new ByteBuffer[16];
+ for(int i = 0; i < dataBufs.length; i++) {
+ dataBufs[i] = newData();
+ }
+
+ // Print header
+ out.printf("\n%s Buffer Performance Table", direct? "Direct": "Non-direct");
+ out.printf(" (bpc: byte-per-crc in MB/sec; #T: #Theads)\n");
+
+ // Warm up implementations to get jit going.
+ final ByteBuffer[] crc32 = {computeCrc(dataBufs[0], 32)};
+ final ByteBuffer[] crc512 = {computeCrc(dataBufs[0], 512)};
+ for (Class<? extends Crc32> c : crcs) {
+ doBench(c, 1, dataBufs, crc32, 32);
+ doBench(c, 1, dataBufs, crc512, 512);
+ }
+
+ // Test on a variety of sizes with different number of threads
+ for (int i = 5; i <= 16; i++) {
+ doBench(crcs, dataBufs, 1 << i, out);
+ }
+ }
+
+ private void doBench(final List<Class<? extends Crc32>> crcs,
+ final ByteBuffer[] dataBufs, final int bytePerCrc, final PrintStream out)
+ throws Exception {
+ final ByteBuffer[] crcBufs = new ByteBuffer[dataBufs.length];
+ for(int i = 0; i < crcBufs.length; i++) {
+ crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc);
+ }
+
+ final String numBytesStr = " bpc ";
+ final String numThreadsStr = "#T";
+ final String diffStr = "% diff";
+
+ out.print('|');
+ printCell(numBytesStr, 0, out);
+ printCell(numThreadsStr, 0, out);
+ for (int i = 0; i < crcs.size(); i++) {
+ final Class<? extends Crc32> c = crcs.get(i);
+ out.print('|');
+ printCell(c.getSimpleName(), 8, out);
+ for(int j = 0; j < i; j++) {
+ printCell(diffStr, diffStr.length(), out);
+ }
+ }
+ out.printf("\n");
+
+ for(int numThreads = 1; numThreads <= dataBufs.length; numThreads <<= 1) {
+ out.printf("|");
+ printCell(String.valueOf(bytePerCrc), numBytesStr.length(), out);
+ printCell(String.valueOf(numThreads), numThreadsStr.length(), out);
+
+ final List<BenchResult> previous = new ArrayList<BenchResult>();
+ for(Class<? extends Crc32> c : crcs) {
+ System.gc();
+
+ final BenchResult result = doBench(c, numThreads, dataBufs, crcBufs,
+ bytePerCrc);
+ printCell(String.format("%9.1f", result.mbps),
+ c.getSimpleName().length() + 1, out);
+
+ //compare result with previous
+ for(BenchResult p : previous) {
+ final double diff = (result.mbps - p.mbps) / p.mbps * 100;
+ printCell(String.format("%5.1f%%", diff), diffStr.length(), out);
+ }
+ previous.add(result);
+ }
+ out.printf("\n");
+ }
+ }
+
+
+ private BenchResult doBench(Class<? extends Crc32> clazz,
+ final int numThreads, final ByteBuffer[] dataBufs,
+ final ByteBuffer[] crcBufs, final int bytePerCrc)
+ throws Exception {
+
+ final Thread[] threads = new Thread[numThreads];
+ final BenchResult[] results = new BenchResult[threads.length];
+
+ {
+ final Constructor<? extends Crc32> ctor = clazz.getConstructor();
+
+ for(int i = 0; i < threads.length; i++) {
+ final Crc32 crc = ctor.newInstance();
+ final long byteProcessed = dataBufs[i].remaining() * trials;
+ final int index = i;
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ final long startTime = System.nanoTime();
+ for (int i = 0; i < trials; i++) {
+ dataBufs[index].mark();
+ crcBufs[index].mark();
+ try {
+ crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index],
+ crc.getClass().getSimpleName(), dataBufs[index].position());
+ } catch (Throwable t) {
+ results[index] = new BenchResult(t);
+ return;
+ } finally {
+ dataBufs[index].reset();
+ crcBufs[index].reset();
+ }
+ }
+ final double secsElapsed = secondsElapsed(startTime);
+ results[index] = new BenchResult(byteProcessed/secsElapsed/MB);
+ }
+ };
+ }
+ }
+
+ for(Thread t : threads) {
+ t.start();
+ }
+ for(Thread t : threads) {
+ t.join();
+ }
+
+ double sum = 0;
+ for(int i = 0; i < results.length; i++) {
+ sum += results[i].getMbps();
+ }
+ return new BenchResult(sum/results.length);
+ }
+
+ private static class BenchResult {
+ /** Speed (MB per second) */
+ final double mbps;
+ final Throwable thrown;
+
+ BenchResult(double mbps) {
+ this.mbps = mbps;
+ this.thrown = null;
+ }
+
+ BenchResult(Throwable e) {
+ this.mbps = 0;
+ this.thrown = e;
+ }
+
+ double getMbps() {
+ if (thrown != null) {
+ throw new AssertionError(thrown);
+ }
+ return mbps;
+ }
+ }
+
+ static double secondsElapsed(final long startTime) {
+ return (System.nanoTime() - startTime) / 1000000000.0d;
+ }
+
+ static void printSystemProperties(PrintStream out) {
+ final String[] names = {
+ "java.version",
+ "java.runtime.name",
+ "java.runtime.version",
+ "java.vm.version",
+ "java.vm.vendor",
+ "java.vm.name",
+ "java.vm.specification.version",
+ "java.specification.version",
+ "os.arch",
+ "os.name",
+ "os.version"
+ };
+ int max = 0;
+ for(String n : names) {
+ if (n.length() > max) {
+ max = n.length();
+ }
+ }
+
+ final Properties p = System.getProperties();
+ for(String n : names) {
+ out.printf("%" + max + "s = %s\n", n, p.getProperty(n));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfaf3c2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
index 73fd25a..8841809 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
@@ -197,4 +197,10 @@ public class TestDataChecksum {
newBuf.limit(dataBuf.limit());
return newBuf;
}
+
+ @Test
+ public void testCrc32() throws Exception {
+ new Crc32PerformanceTest(8, 3, true).run();
+ new Crc32PerformanceTest(8, 3, false).run();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfaf3c2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 1f783f6..62dfe79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -52,6 +52,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -62,8 +63,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.client.HdfsUtils;
+import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -929,11 +930,12 @@ public class TestDFSClientRetries {
try {
dis.read(arr, 0, (int)FILE_LENGTH);
fail("Expected ChecksumException not thrown");
- } catch (Exception ex) {
+ } catch (ChecksumException ex) {
GenericTestUtils.assertExceptionContains(
- "Checksum error", ex);
+ "Checksum", ex);
}
}
+ client.close();
} finally {
cluster.shutdown();
}
[04/50] [abbrv] hadoop git commit: HDFS-9691.
TestBlockManagerSafeMode#testCheckSafeMode fails intermittently. Contributed
by Mingliang Liu.
Posted by ar...@apache.org.
HDFS-9691. TestBlockManagerSafeMode#testCheckSafeMode fails intermittently. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac5da113
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac5da113
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac5da113
Branch: refs/heads/HDFS-1312
Commit: ac5da11399525dd31c49700a0bbf01cd37e347d5
Parents: 1c48e50
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 15 14:00:09 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 15 14:00:09 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/blockmanagement/TestBlockManagerSafeMode.java | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac5da113/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eeff967..eb2b21e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1015,6 +1015,9 @@ Release 2.9.0 - UNRELEASED
HDFS-9621. getListing wrongly associates Erasure Coding policy to pre-existing
replicated files under an EC directory. (jing9)
+ HDFS-9691. TestBlockManagerSafeMode#testCheckSafeMode fails intermittently.
+ (Mingliang Liu via aajisaka)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac5da113/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index ade5574..d9da2c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -137,30 +137,30 @@ public class TestBlockManagerSafeMode {
}
// PENDING_THRESHOLD -> EXTENSION
+ Whitebox.setInternalState(bmSafeMode, "extension", Integer.MAX_VALUE);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
setBlockSafe(BLOCK_THRESHOLD);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus());
- Whitebox.setInternalState(bmSafeMode, "smmthread", null);
// PENDING_THRESHOLD -> OFF
+ Whitebox.setInternalState(bmSafeMode, "extension", 0);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
setBlockSafe(BLOCK_THRESHOLD);
- Whitebox.setInternalState(bmSafeMode, "extension", 0);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.OFF, getSafeModeStatus());
// stays in EXTENSION
- setSafeModeStatus(BMSafeModeStatus.EXTENSION);
setBlockSafe(0);
+ setSafeModeStatus(BMSafeModeStatus.EXTENSION);
Whitebox.setInternalState(bmSafeMode, "extension", 0);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus());
// stays in EXTENSION: pending extension period
+ Whitebox.setInternalState(bmSafeMode, "extension", Integer.MAX_VALUE);
setSafeModeStatus(BMSafeModeStatus.EXTENSION);
setBlockSafe(BLOCK_THRESHOLD);
- Whitebox.setInternalState(bmSafeMode, "extension", Integer.MAX_VALUE);
bmSafeMode.checkSafeMode();
assertEquals(BMSafeModeStatus.EXTENSION, getSafeModeStatus());
}
[42/50] [abbrv] hadoop git commit: HDFS-9839. Reduce verbosity of
processReport logging. (Contributed by Arpit Agarwal)
Posted by ar...@apache.org.
HDFS-9839. Reduce verbosity of processReport logging. (Contributed by Arpit Agarwal)
This closes #78
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5abd293
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5abd293
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5abd293
Branch: refs/heads/HDFS-1312
Commit: d5abd293a890a8a1da48a166a291ae1c5644ad57
Parents: e54cc29
Author: Arpit Agarwal <ar...@apache.org>
Authored: Sat Feb 20 23:19:09 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Sat Feb 20 23:19:09 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hdfs/server/blockmanagement/BlockManager.java | 18 +++++++++---------
2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5abd293/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1d0379c..9215488 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2802,6 +2802,8 @@ Release 2.8.0 - UNRELEASED
HDFS-6832. Fix the usage of 'hdfs namenode' command.
(Manjunath Ballur via aajisaka)
+ HDFS-9839. Reduce verbosity of processReport logging. (Arpit Agarwal)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5abd293/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index cc52b6e..eaebed9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2060,7 +2060,7 @@ public class BlockManager implements BlockStatsMXBean {
final long startTime = Time.monotonicNow(); //after acquiring write lock
final long endTime;
DatanodeDescriptor node;
- Collection<Block> invalidatedBlocks = null;
+ Collection<Block> invalidatedBlocks = Collections.emptyList();
try {
node = datanodeManager.getDatanode(nodeID);
@@ -2136,11 +2136,9 @@ public class BlockManager implements BlockStatsMXBean {
namesystem.writeUnlock();
}
- if (invalidatedBlocks != null) {
- for (Block b : invalidatedBlocks) {
- blockLog.info("BLOCK* processReport: {} on node {} size {} does not " +
- "belong to any file", b, node, b.getNumBytes());
- }
+ for (Block b : invalidatedBlocks) {
+ blockLog.debug("BLOCK* processReport: {} on node {} size {} does not " +
+ "belong to any file", b, node, b.getNumBytes());
}
// Log the block report processing stats from Namenode perspective
@@ -2149,9 +2147,11 @@ public class BlockManager implements BlockStatsMXBean {
metrics.addBlockReport((int) (endTime - startTime));
}
blockLog.info("BLOCK* processReport: from storage {} node {}, " +
- "blocks: {}, hasStaleStorage: {}, processing time: {} msecs", storage
- .getStorageID(), nodeID, newReport.getNumberOfBlocks(),
- node.hasStaleStorages(), (endTime - startTime));
+ "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " +
+ "invalidatedBlocks: {}", storage.getStorageID(), nodeID,
+ newReport.getNumberOfBlocks(),
+ node.hasStaleStorages(), (endTime - startTime),
+ invalidatedBlocks.size());
return !node.hasStaleStorages();
}
[45/50] [abbrv] hadoop git commit: HDFS-9425. Expose number of blocks
per volume as a metric (Contributed by Brahma Reddy Battula)
Posted by ar...@apache.org.
HDFS-9425. Expose number of blocks per volume as a metric (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/342c9572
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/342c9572
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/342c9572
Branch: refs/heads/HDFS-1312
Commit: 342c9572bf6a623287f34c5cc0bc3be6038c191a
Parents: f313516
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Feb 22 09:29:23 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Feb 22 09:29:23 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../datanode/fsdataset/impl/BlockPoolSlice.java | 29 ++++++++++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 12 ++++-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 30 +++++++++--
.../server/datanode/TestDataNodeMXBean.java | 55 ++++++++++++++++++++
5 files changed, 121 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c9572/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9a156a5..3552c8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2024,6 +2024,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9768. Reuse ObjectMapper instance in HDFS to improve the performance.
(Lin Yiqun via aajisaka)
+ HDFS-9425. Expose number of blocks per volume as a metric
+ (Brahma Reddy Battula via vinayakumarb)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c9572/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 188ab68..aa33851 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -30,6 +30,7 @@ import java.io.RandomAccessFile;
import java.io.Writer;
import java.util.Iterator;
import java.util.Scanner;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
@@ -86,6 +87,7 @@ class BlockPoolSlice {
private final boolean deleteDuplicateReplicas;
private static final String REPLICA_CACHE_FILE = "replicas";
private final long replicaCacheExpiry = 5*60*1000;
+ private AtomicLong numOfBlocks = new AtomicLong();
private final long cachedDfsUsedCheckTime;
private final Timer timer;
@@ -273,7 +275,11 @@ class BlockPoolSlice {
*/
File createTmpFile(Block b) throws IOException {
File f = new File(tmpDir, b.getBlockName());
- return DatanodeUtil.createTmpFile(b, f);
+ File tmpFile = DatanodeUtil.createTmpFile(b, f);
+ // If any exception during creation, its expected that counter will not be
+ // incremented, So no need to decrement
+ incrNumBlocks();
+ return tmpFile;
}
/**
@@ -282,7 +288,11 @@ class BlockPoolSlice {
*/
File createRbwFile(Block b) throws IOException {
File f = new File(rbwDir, b.getBlockName());
- return DatanodeUtil.createTmpFile(b, f);
+ File rbwFile = DatanodeUtil.createTmpFile(b, f);
+ // If any exception during creation, its expected that counter will not be
+ // incremented, So no need to decrement
+ incrNumBlocks();
+ return rbwFile;
}
File addFinalizedBlock(Block b, File f) throws IOException {
@@ -493,6 +503,9 @@ class BlockPoolSlice {
} else {
lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
}
+ if (oldReplica == null) {
+ incrNumBlocks();
+ }
}
@@ -825,4 +838,16 @@ class BlockPoolSlice {
}
}
}
+
+ void incrNumBlocks() {
+ numOfBlocks.incrementAndGet();
+ }
+
+ void decrNumBlocks() {
+ numOfBlocks.decrementAndGet();
+ }
+
+ public long getNumOfBlocks() {
+ return numOfBlocks.get();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c9572/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 819d097..d1cb836 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -965,6 +965,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
newReplicaInfo.setNumBytes(blockFiles[1].length());
// Finalize the copied files
newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
+ synchronized (this) {
+ // Increment numBlocks here as this block moved without knowing to BPS
+ FsVolumeImpl volume = (FsVolumeImpl) newReplicaInfo.getVolume();
+ volume.getBlockPoolSlice(block.getBlockPoolId()).incrNumBlocks();
+ }
removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
@@ -2599,6 +2604,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final long reservedSpace; // size of space reserved for non-HDFS
final long reservedSpaceForReplicas; // size of space reserved RBW or
// re-replication
+ final long numBlocks;
VolumeInfo(FsVolumeImpl v, long usedSpace, long freeSpace) {
this.directory = v.toString();
@@ -2606,6 +2612,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
this.freeSpace = freeSpace;
this.reservedSpace = v.getReserved();
this.reservedSpaceForReplicas = v.getReservedForReplicas();
+ this.numBlocks = v.getNumBlocks();
}
}
@@ -2640,6 +2647,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
innerInfo.put("freeSpace", v.freeSpace);
innerInfo.put("reservedSpace", v.reservedSpace);
innerInfo.put("reservedSpaceForReplicas", v.reservedSpaceForReplicas);
+ innerInfo.put("numBlocks", v.numBlocks);
info.put(v.directory, innerInfo);
}
return info;
@@ -2728,8 +2736,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
synchronized (FsDatasetImpl.this) {
ramDiskReplicaTracker.recordEndLazyPersist(bpId, blockId, savedFiles);
- targetVolume.incDfsUsed(bpId,
- savedFiles[0].length() + savedFiles[1].length());
+ targetVolume.incDfsUsedAndNumBlocks(bpId, savedFiles[0].length()
+ + savedFiles[1].length());
// Update metrics (ignore the metadata file size)
datanode.getMetrics().incrRamDiskBlocksLazyPersisted();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c9572/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 1533a32..857d0ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -284,21 +284,35 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
void onBlockFileDeletion(String bpid, long value) {
- decDfsUsed(bpid, value);
+ decDfsUsedAndNumBlocks(bpid, value, true);
if (isTransientStorage()) {
dataset.releaseLockedMemory(value, true);
}
}
void onMetaFileDeletion(String bpid, long value) {
- decDfsUsed(bpid, value);
+ decDfsUsedAndNumBlocks(bpid, value, false);
}
- private void decDfsUsed(String bpid, long value) {
+ private void decDfsUsedAndNumBlocks(String bpid, long value,
+ boolean blockFileDeleted) {
synchronized(dataset) {
BlockPoolSlice bp = bpSlices.get(bpid);
if (bp != null) {
bp.decDfsUsed(value);
+ if (blockFileDeleted) {
+ bp.decrNumBlocks();
+ }
+ }
+ }
+ }
+
+ void incDfsUsedAndNumBlocks(String bpid, long value) {
+ synchronized (dataset) {
+ BlockPoolSlice bp = bpSlices.get(bpid);
+ if (bp != null) {
+ bp.incDfsUsed(value);
+ bp.incrNumBlocks();
}
}
}
@@ -847,7 +861,15 @@ public class FsVolumeImpl implements FsVolumeSpi {
throws IOException {
getBlockPoolSlice(bpid).getVolumeMap(volumeMap, ramDiskReplicaMap);
}
-
+
+ long getNumBlocks() {
+ long numBlocks = 0;
+ for (BlockPoolSlice s : bpSlices.values()) {
+ numBlocks += s.getNumOfBlocks();
+ }
+ return numBlocks;
+ }
+
@Override
public String toString() {
return currentDir.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/342c9572/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
index a7d0a5e..6d520ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
@@ -18,15 +18,23 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.lang.management.ManagementFactory;
+import java.util.Collection;
import java.util.List;
+import java.util.Map;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.Assert;
import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
+
+import static org.junit.Assert.assertEquals;
/**
* Class for testing {@link DataNodeMXBean} implementation
@@ -84,4 +92,51 @@ public class TestDataNodeMXBean {
private static String replaceDigits(final String s) {
return s.replaceAll("[0-9]+", "_DIGITS_");
}
+
+ @Test
+ public void testDataNodeMXBeanBlockCount() throws Exception {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+
+ try {
+ List<DataNode> datanodes = cluster.getDataNodes();
+ assertEquals(datanodes.size(), 1);
+
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName =
+ new ObjectName("Hadoop:service=DataNode,name=DataNodeInfo");
+ FileSystem fs = cluster.getFileSystem();
+ for (int i = 0; i < 5; i++) {
+ DFSTestUtil.createFile(fs, new Path("/tmp.txt" + i), 1024, (short) 1,
+ 1L);
+ }
+ assertEquals("Before restart DN", 5, getTotalNumBlocks(mbs, mxbeanName));
+ cluster.restartDataNode(0);
+ cluster.waitActive();
+ assertEquals("After restart DN", 5, getTotalNumBlocks(mbs, mxbeanName));
+ fs.delete(new Path("/tmp.txt1"), true);
+ // Wait till replica gets deleted on disk.
+ Thread.sleep(5000);
+ assertEquals("After delete one file", 4,
+ getTotalNumBlocks(mbs, mxbeanName));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ int getTotalNumBlocks(MBeanServer mbs, ObjectName mxbeanName)
+ throws Exception {
+ int totalBlocks = 0;
+ String volumeInfo = (String) mbs.getAttribute(mxbeanName, "VolumeInfo");
+ Map<?, ?> m = (Map<?, ?>) JSON.parse(volumeInfo);
+ Collection<Map<String, Long>> values =
+ (Collection<Map<String, Long>>) m.values();
+ for (Map<String, Long> volumeInfoMap : values) {
+ totalBlocks += volumeInfoMap.get("numBlocks");
+ }
+ return totalBlocks;
+ }
}
[09/50] [abbrv] hadoop git commit: HADOOP-12810.
FileSystem#listLocatedStatus causes unnecessary RPC calls (Contributed by
Ryan Blue)
Posted by ar...@apache.org.
HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls (Contributed by Ryan Blue)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96ea3094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96ea3094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96ea3094
Branch: refs/heads/HDFS-1312
Commit: 96ea3094315bb1e1a5e268e3817c7fdedc3e9462
Parents: 77ba5ad
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Feb 17 10:13:41 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Feb 17 10:13:41 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96ea3094/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1a2b32f..0aa39d9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1720,6 +1720,9 @@ Release 2.7.3 - UNRELEASED
OPTIMIZATIONS
+ HADOOP-12810. FileSystem#listLocatedStatus causes unnecessary RPC calls
+ (Ryan Blue via vinayakumarb)
+
BUG FIXES
HADOOP-12296. when setnetgrent returns 0 in linux, exception should be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96ea3094/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 920c5a6..f4a2e7d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1749,8 +1749,10 @@ public abstract class FileSystem extends Configured implements Closeable {
throw new NoSuchElementException("No more entries in " + f);
}
FileStatus result = stats[i++];
+ // for files, use getBlockLocations(FileStatus, int, int) to avoid
+ // calling getFileStatus(Path) to load the FileStatus again
BlockLocation[] locs = result.isFile() ?
- getFileBlockLocations(result.getPath(), 0, result.getLen()) :
+ getFileBlockLocations(result, 0, result.getLen()) :
null;
return new LocatedFileStatus(result, locs);
}
[11/50] [abbrv] hadoop git commit: MAPREDUCE-6341. addendum patch to
fix typos. Contributed by John Michael Luy.
Posted by ar...@apache.org.
MAPREDUCE-6341. addendum patch to fix typos. Contributed by John Michael Luy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/453e7e0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/453e7e0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/453e7e0d
Branch: refs/heads/HDFS-1312
Commit: 453e7e0d1091cf795152c8992c6c5cbc772d80d6
Parents: 2536ece
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Feb 17 14:13:31 2016 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Feb 17 14:17:32 2016 +0900
----------------------------------------------------------------------
.../src/site/markdown/MapReduceTutorial.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453e7e0d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
index 74c6c66..16f3afb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
@@ -319,7 +319,7 @@ Thus the output of the job is:
< Goodbye, 1>
< Hadoop, 2>
< Hello, 2>
- < World, 2>`
+ < World, 2>
The `main` method specifies various facets of the job, such as the input/output paths (passed via the command line), key/value types, input/output formats etc., in the `Job`. It then calls the `job.waitForCompletion` to submit the job and monitor its progress.
@@ -765,7 +765,7 @@ RecordWriter implementations write the job outputs to the `FileSystem`.
Users submit jobs to Queues. Queues, as collection of jobs, allow the system to provide specific functionality. For example, queues use ACLs to control which users who can submit jobs to them. Queues are expected to be primarily used by Hadoop Schedulers.
-Hadoop comes configured with a single mandatory queue, called 'default'. Queue names are defined in the `mapreduce.job.queuename`\> property of the Hadoop site configuration. Some job schedulers, such as the
+Hadoop comes configured with a single mandatory queue, called 'default'. Queue names are defined in the `mapreduce.job.queuename` property of the Hadoop site configuration. Some job schedulers, such as the
[Capacity Scheduler](../../hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html),
support multiple queues.
@@ -799,7 +799,7 @@ The files/archives can be distributed by setting the property `mapreduce.job.cac
and
[Job.setCacheFiles(URI[])](../../api/org/apache/hadoop/mapreduce/Job.html)/
[Job.setCacheArchives(URI[])](../../api/org/apache/hadoop/mapreduce/Job.html)
-where URI is of the form `hdfs://host:port/absolute-path\#link-name`. In Streaming, the files can be distributed through command line option `-cacheFile/-cacheArchive`.
+where URI is of the form `hdfs://host:port/absolute-path#link-name`. In Streaming, the files can be distributed through command line option `-cacheFile/-cacheArchive`.
The `DistributedCache` can also be used as a rudimentary software distribution mechanism for use in the map and/or reduce tasks. It can be used to distribute both jars and native libraries. The
[Job.addArchiveToClassPath(Path)](../../api/org/apache/hadoop/mapreduce/Job.html) or
[25/50] [abbrv] hadoop git commit: HDFS-9711. Integrate CSRF
prevention filter in WebHDFS. Contributed by Chris Nauroth.
Posted by ar...@apache.org.
HDFS-9711. Integrate CSRF prevention filter in WebHDFS. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d1889a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d1889a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d1889a6
Branch: refs/heads/HDFS-1312
Commit: 5d1889a66d91608d34ca9411fb6e9161e637e9d3
Parents: e78d8e6
Author: cnauroth <cn...@apache.org>
Authored: Thu Feb 18 10:07:28 2016 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Thu Feb 18 10:07:28 2016 -0800
----------------------------------------------------------------------
.../security/http/RestCsrfPreventionFilter.java | 168 +++++++++++++++++--
.../hdfs/client/HdfsClientConfigKeys.java | 12 ++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 59 +++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/datanode/web/DatanodeHttpServer.java | 114 ++++++++++++-
.../web/PortUnificationServerHandler.java | 18 +-
.../web/RestCsrfPreventionFilterHandler.java | 137 +++++++++++++++
.../server/namenode/NameNodeHttpServer.java | 15 +-
.../src/main/resources/hdfs-default.xml | 52 ++++++
.../src/main/webapps/hdfs/explorer.html | 1 +
.../src/main/webapps/static/rest-csrf.js | 91 ++++++++++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 36 ++++
...TestWebHdfsWithRestCsrfPreventionFilter.java | 166 ++++++++++++++++++
13 files changed, 851 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
index 4f7f5bb..c0f7e39 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.security.http;
import java.io.IOException;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Set;
@@ -32,6 +34,13 @@ import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* This filter provides protection against cross site request forgery (CSRF)
* attacks for REST APIs. Enabling this filter on an endpoint results in the
@@ -39,7 +48,13 @@ import javax.servlet.http.HttpServletResponse;
* with every request. In the absense of this header the filter will reject the
* attempt as a bad request.
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class RestCsrfPreventionFilter implements Filter {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RestCsrfPreventionFilter.class);
+
public static final String HEADER_USER_AGENT = "User-Agent";
public static final String BROWSER_USER_AGENT_PARAM =
"browser-useragents-regex";
@@ -72,6 +87,9 @@ public class RestCsrfPreventionFilter implements Filter {
agents = BROWSER_USER_AGENTS_DEFAULT;
}
parseBrowserUserAgents(agents);
+ LOG.info("Adding cross-site request forgery (CSRF) protection, "
+ + "headerName = {}, methodsToIgnore = {}, browserUserAgents = {}",
+ headerName, methodsToIgnore, browserUserAgents);
}
void parseBrowserUserAgents(String userAgents) {
@@ -118,22 +136,152 @@ public class RestCsrfPreventionFilter implements Filter {
return false;
}
- @Override
- public void doFilter(ServletRequest request, ServletResponse response,
- FilterChain chain) throws IOException, ServletException {
- HttpServletRequest httpRequest = (HttpServletRequest)request;
- if (!isBrowser(httpRequest.getHeader(HEADER_USER_AGENT)) ||
- methodsToIgnore.contains(httpRequest.getMethod()) ||
- httpRequest.getHeader(headerName) != null) {
- chain.doFilter(request, response);
+ /**
+ * Defines the minimal API requirements for the filter to execute its
+ * filtering logic. This interface exists to facilitate integration in
+ * components that do not run within a servlet container and therefore cannot
+ * rely on a servlet container to dispatch to the {@link #doFilter} method.
+ * Applications that do run inside a servlet container will not need to write
+ * code that uses this interface. Instead, they can use typical servlet
+ * container configuration mechanisms to insert the filter.
+ */
+ public interface HttpInteraction {
+
+ /**
+ * Returns the value of a header.
+ *
+ * @param header name of header
+ * @return value of header
+ */
+ String getHeader(String header);
+
+ /**
+ * Returns the method.
+ *
+ * @return method
+ */
+ String getMethod();
+
+ /**
+ * Called by the filter after it decides that the request may proceed.
+ *
+ * @throws IOException if there is an I/O error
+ * @throws ServletException if the implementation relies on the servlet API
+ * and a servlet API call has failed
+ */
+ void proceed() throws IOException, ServletException;
+
+ /**
+ * Called by the filter after it decides that the request is a potential
+ * CSRF attack and therefore must be rejected.
+ *
+ * @param code status code to send
+ * @param message response message
+ * @throws IOException if there is an I/O error
+ */
+ void sendError(int code, String message) throws IOException;
+ }
+
+ /**
+ * Handles an {@link HttpInteraction} by applying the filtering logic.
+ *
+ * @param httpInteraction caller's HTTP interaction
+ * @throws IOException if there is an I/O error
+ * @throws ServletException if the implementation relies on the servlet API
+ * and a servlet API call has failed
+ */
+ public void handleHttpInteraction(HttpInteraction httpInteraction)
+ throws IOException, ServletException {
+ if (!isBrowser(httpInteraction.getHeader(HEADER_USER_AGENT)) ||
+ methodsToIgnore.contains(httpInteraction.getMethod()) ||
+ httpInteraction.getHeader(headerName) != null) {
+ httpInteraction.proceed();
} else {
- ((HttpServletResponse)response).sendError(
- HttpServletResponse.SC_BAD_REQUEST,
+ httpInteraction.sendError(HttpServletResponse.SC_BAD_REQUEST,
"Missing Required Header for CSRF Vulnerability Protection");
}
}
@Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ final FilterChain chain) throws IOException, ServletException {
+ final HttpServletRequest httpRequest = (HttpServletRequest)request;
+ final HttpServletResponse httpResponse = (HttpServletResponse)response;
+ handleHttpInteraction(new ServletFilterHttpInteraction(httpRequest,
+ httpResponse, chain));
+ }
+
+ @Override
public void destroy() {
}
+
+ /**
+ * Constructs a mapping of configuration properties to be used for filter
+ * initialization. The mapping includes all properties that start with the
+ * specified configuration prefix. Property names in the mapping are trimmed
+ * to remove the configuration prefix.
+ *
+ * @param conf configuration to read
+ * @param confPrefix configuration prefix
+ * @return mapping of configuration properties to be used for filter
+ * initialization
+ */
+ public static Map<String, String> getFilterParams(Configuration conf,
+ String confPrefix) {
+ Map<String, String> filterConfigMap = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf) {
+ String name = entry.getKey();
+ if (name.startsWith(confPrefix)) {
+ String value = conf.get(name);
+ name = name.substring(confPrefix.length());
+ filterConfigMap.put(name, value);
+ }
+ }
+ return filterConfigMap;
+ }
+
+ /**
+ * {@link HttpInteraction} implementation for use in the servlet filter.
+ */
+ private static final class ServletFilterHttpInteraction
+ implements HttpInteraction {
+
+ private final FilterChain chain;
+ private final HttpServletRequest httpRequest;
+ private final HttpServletResponse httpResponse;
+
+ /**
+ * Creates a new ServletFilterHttpInteraction.
+ *
+ * @param httpRequest request to process
+ * @param httpResponse response to process
+ * @param chain filter chain to forward to if HTTP interaction is allowed
+ */
+ public ServletFilterHttpInteraction(HttpServletRequest httpRequest,
+ HttpServletResponse httpResponse, FilterChain chain) {
+ this.httpRequest = httpRequest;
+ this.httpResponse = httpResponse;
+ this.chain = chain;
+ }
+
+ @Override
+ public String getHeader(String header) {
+ return httpRequest.getHeader(header);
+ }
+
+ @Override
+ public String getMethod() {
+ return httpRequest.getMethod();
+ }
+
+ @Override
+ public void proceed() throws IOException, ServletException {
+ chain.doFilter(httpRequest, httpResponse);
+ }
+
+ @Override
+ public void sendError(int code, String message) throws IOException {
+ httpResponse.sendError(code, message);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 775907e..ce0f3e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -41,6 +41,18 @@ public interface HdfsClientConfigKeys {
String DFS_WEBHDFS_OAUTH_ENABLED_KEY = "dfs.webhdfs.oauth2.enabled";
boolean DFS_WEBHDFS_OAUTH_ENABLED_DEFAULT = false;
+ String DFS_WEBHDFS_REST_CSRF_ENABLED_KEY = "dfs.webhdfs.rest-csrf.enabled";
+ boolean DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT = false;
+ String DFS_WEBHDFS_REST_CSRF_CUSTOM_HEADER_KEY =
+ "dfs.webhdfs.rest-csrf.custom-header";
+ String DFS_WEBHDFS_REST_CSRF_CUSTOM_HEADER_DEFAULT = "X-XSRF-HEADER";
+ String DFS_WEBHDFS_REST_CSRF_METHODS_TO_IGNORE_KEY =
+ "dfs.webhdfs.rest-csrf.methods-to-ignore";
+ String DFS_WEBHDFS_REST_CSRF_METHODS_TO_IGNORE_DEFAULT =
+ "GET,OPTIONS,HEAD,TRACE";
+ String DFS_WEBHDFS_REST_CSRF_BROWSER_USERAGENTS_REGEX_KEY =
+ "dfs.webhdfs.rest-csrf.browser-useragents-regex";
+
String OAUTH_CLIENT_ID_KEY = "dfs.webhdfs.oauth2.client.id";
String OAUTH_REFRESH_URL_KEY = "dfs.webhdfs.oauth2.refresh.url";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 82cf655..3527c27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -18,6 +18,13 @@
package org.apache.hadoop.hdfs.web;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_CUSTOM_HEADER_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_CUSTOM_HEADER_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_METHODS_TO_IGNORE_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_METHODS_TO_IGNORE_KEY;
+
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.EOFException;
@@ -33,8 +40,10 @@ import java.net.URL;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.StringTokenizer;
import javax.ws.rs.core.HttpHeaders;
@@ -129,6 +138,8 @@ public class WebHdfsFileSystem extends FileSystem
private InetSocketAddress nnAddrs[];
private int currentNNAddrIndex;
private boolean disallowFallbackToInsecureCluster;
+ private String restCsrfCustomHeader;
+ private Set<String> restCsrfMethodsToIgnore;
private static final ObjectReader READER =
new ObjectMapper().reader(Map.class);
@@ -227,9 +238,52 @@ public class WebHdfsFileSystem extends FileSystem
this.disallowFallbackToInsecureCluster = !conf.getBoolean(
CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
+ this.initializeRestCsrf(conf);
this.delegationToken = null;
}
+ /**
+ * Initializes client-side handling of cross-site request forgery (CSRF)
+ * protection by figuring out the custom HTTP headers that need to be sent in
+ * requests and which HTTP methods are ignored because they do not require
+ * CSRF protection.
+ *
+ * @param conf configuration to read
+ */
+ private void initializeRestCsrf(Configuration conf) {
+ if (conf.getBoolean(DFS_WEBHDFS_REST_CSRF_ENABLED_KEY,
+ DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT)) {
+ this.restCsrfCustomHeader = conf.getTrimmed(
+ DFS_WEBHDFS_REST_CSRF_CUSTOM_HEADER_KEY,
+ DFS_WEBHDFS_REST_CSRF_CUSTOM_HEADER_DEFAULT);
+ this.restCsrfMethodsToIgnore = new HashSet<>();
+ this.restCsrfMethodsToIgnore.addAll(getTrimmedStringList(conf,
+ DFS_WEBHDFS_REST_CSRF_METHODS_TO_IGNORE_KEY,
+ DFS_WEBHDFS_REST_CSRF_METHODS_TO_IGNORE_DEFAULT));
+ } else {
+ this.restCsrfCustomHeader = null;
+ this.restCsrfMethodsToIgnore = null;
+ }
+ }
+
+ /**
+ * Returns a list of strings from a comma-delimited configuration value.
+ *
+ * @param conf configuration to check
+ * @param name configuration property name
+ * @param defaultValue default value if no value found for name
+ * @return list of strings from comma-delimited configuration value, or an
+ * empty list if not found
+ */
+ private static List<String> getTrimmedStringList(Configuration conf,
+ String name, String defaultValue) {
+ String valueString = conf.get(name, defaultValue);
+ if (valueString == null) {
+ return new ArrayList<>();
+ }
+ return new ArrayList<>(StringUtils.getTrimmedStringCollection(valueString));
+ }
+
@Override
public URI getCanonicalUri() {
return super.getCanonicalUri();
@@ -601,6 +655,11 @@ public class WebHdfsFileSystem extends FileSystem
final boolean doOutput = op.getDoOutput();
conn.setRequestMethod(op.getType().toString());
conn.setInstanceFollowRedirects(false);
+ if (restCsrfCustomHeader != null &&
+ !restCsrfMethodsToIgnore.contains(op.getType().name())) {
+ // The value of the header is unimportant. Only its presence matters.
+ conn.setRequestProperty(restCsrfCustomHeader, "\"\"");
+ }
switch (op.getType()) {
// if not sending a message body for a POST or PUT operation, need
// to ensure the server/proxy knows this
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 61a783d..2948a94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1041,6 +1041,8 @@ Release 2.8.0 - UNRELEASED
HDFS-9244. Support nested encryption zones. (zhz)
+ HDFS-9711. Integrate CSRF prevention filter in WebHDFS. (cnauroth)
+
IMPROVEMENTS
HDFS-9257. improve error message for "Absolute path required" in INode.java
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index fc24fae..f9bdbf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -17,6 +17,15 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_KEY;
+
+import java.util.Enumeration;
+import java.util.Map;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFactory;
import io.netty.channel.ChannelFuture;
@@ -46,6 +55,7 @@ import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
import org.apache.hadoop.security.ssl.SSLFactory;
import java.io.Closeable;
@@ -73,6 +83,7 @@ public class DatanodeHttpServer implements Closeable {
private final ServerBootstrap httpsServer;
private final Configuration conf;
private final Configuration confForCreate;
+ private final RestCsrfPreventionFilter restCsrfPreventionFilter;
private InetSocketAddress httpAddress;
private InetSocketAddress httpsAddress;
static final Log LOG = LogFactory.getLog(DatanodeHttpServer.class);
@@ -81,6 +92,7 @@ public class DatanodeHttpServer implements Closeable {
final DataNode datanode,
final ServerSocketChannel externalHttpChannel)
throws IOException {
+ this.restCsrfPreventionFilter = createRestCsrfPreventionFilter(conf);
this.conf = conf;
Configuration confForInfoServer = new Configuration(conf);
@@ -117,7 +129,7 @@ public class DatanodeHttpServer implements Closeable {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ch.pipeline().addLast(new PortUnificationServerHandler(jettyAddr,
- conf, confForCreate));
+ conf, confForCreate, restCsrfPreventionFilter));
}
});
@@ -165,11 +177,16 @@ public class DatanodeHttpServer implements Closeable {
protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline p = ch.pipeline();
p.addLast(
- new SslHandler(sslFactory.createSSLEngine()),
- new HttpRequestDecoder(),
- new HttpResponseEncoder(),
- new ChunkedWriteHandler(),
- new URLDispatcher(jettyAddr, conf, confForCreate));
+ new SslHandler(sslFactory.createSSLEngine()),
+ new HttpRequestDecoder(),
+ new HttpResponseEncoder());
+ if (restCsrfPreventionFilter != null) {
+ p.addLast(new RestCsrfPreventionFilterHandler(
+ restCsrfPreventionFilter));
+ }
+ p.addLast(
+ new ChunkedWriteHandler(),
+ new URLDispatcher(jettyAddr, conf, confForCreate));
}
});
} else {
@@ -252,4 +269,87 @@ public class DatanodeHttpServer implements Closeable {
InetSocketAddress inetSocker = NetUtils.createSocketAddr(addr);
return inetSocker.getHostString();
}
-}
\ No newline at end of file
+
+ /**
+ * Creates the {@link RestCsrfPreventionFilter} for the DataNode. Since the
+ * DataNode HTTP server is not implemented in terms of the servlet API, it
+ * takes some extra effort to obtain an instance of the filter. This method
+ * takes care of configuration and implementing just enough of the servlet API
+ * and related interfaces so that the DataNode can get a fully initialized
+ * instance of the filter.
+ *
+ * @param conf configuration to read
+ * @return initialized filter, or null if CSRF protection not enabled
+ */
+ private static RestCsrfPreventionFilter createRestCsrfPreventionFilter(
+ Configuration conf) {
+ if (!conf.getBoolean(DFS_WEBHDFS_REST_CSRF_ENABLED_KEY,
+ DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT)) {
+ return null;
+ }
+ String restCsrfClassName = RestCsrfPreventionFilter.class.getName();
+ Map<String, String> restCsrfParams = RestCsrfPreventionFilter
+ .getFilterParams(conf, "dfs.webhdfs.rest-csrf.");
+ RestCsrfPreventionFilter filter = new RestCsrfPreventionFilter();
+ try {
+ filter.init(new MapBasedFilterConfig(restCsrfClassName, restCsrfParams));
+ } catch (ServletException e) {
+ throw new IllegalStateException(
+ "Failed to initialize RestCsrfPreventionFilter.", e);
+ }
+ return filter;
+ }
+
+ /**
+ * A minimal {@link FilterConfig} implementation backed by a {@link Map}.
+ */
+ private static final class MapBasedFilterConfig implements FilterConfig {
+
+ private final String filterName;
+ private final Map<String, String> parameters;
+
+ /**
+ * Creates a new MapBasedFilterConfig.
+ *
+ * @param filterName filter name
+ * @param parameters mapping of filter initialization parameters
+ */
+ public MapBasedFilterConfig(String filterName,
+ Map<String, String> parameters) {
+ this.filterName = filterName;
+ this.parameters = parameters;
+ }
+
+ @Override
+ public String getFilterName() {
+ return this.filterName;
+ }
+
+ @Override
+ public String getInitParameter(String name) {
+ return this.parameters.get(name);
+ }
+
+ @Override
+ public Enumeration<String> getInitParameterNames() {
+ throw this.notImplemented();
+ }
+
+ @Override
+ public ServletContext getServletContext() {
+ throw this.notImplemented();
+ }
+
+ /**
+ * Creates an exception indicating that an interface method is not
+ * implemented. These should never be seen in practice, because it is only
+ * used for methods that are not called by {@link RestCsrfPreventionFilter}.
+ *
+ * @return exception indicating method not implemented
+ */
+ private UnsupportedOperationException notImplemented() {
+ return new UnsupportedOperationException(this.getClass().getSimpleName()
+ + " does not implement this method.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
index 7ebc070..ff10c6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.web.dtp.DtpHttp2Handler;
+import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
@@ -51,19 +52,32 @@ public class PortUnificationServerHandler extends ByteToMessageDecoder {
private final Configuration confForCreate;
+ private final RestCsrfPreventionFilter restCsrfPreventionFilter;
+
public PortUnificationServerHandler(InetSocketAddress proxyHost,
- Configuration conf, Configuration confForCreate) {
+ Configuration conf, Configuration confForCreate,
+ RestCsrfPreventionFilter restCsrfPreventionFilter) {
this.proxyHost = proxyHost;
this.conf = conf;
this.confForCreate = confForCreate;
+ this.restCsrfPreventionFilter = restCsrfPreventionFilter;
}
private void configureHttp1(ChannelHandlerContext ctx) {
- ctx.pipeline().addLast(new HttpServerCodec(), new ChunkedWriteHandler(),
+ ctx.pipeline().addLast(new HttpServerCodec());
+ if (this.restCsrfPreventionFilter != null) {
+ ctx.pipeline().addLast(new RestCsrfPreventionFilterHandler(
+ this.restCsrfPreventionFilter));
+ }
+ ctx.pipeline().addLast(new ChunkedWriteHandler(),
new URLDispatcher(proxyHost, conf, confForCreate));
}
private void configureHttp2(ChannelHandlerContext ctx) {
+ if (this.restCsrfPreventionFilter != null) {
+ ctx.pipeline().addLast(new RestCsrfPreventionFilterHandler(
+ this.restCsrfPreventionFilter));
+ }
ctx.pipeline().addLast(new DtpHttp2Handler());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
new file mode 100644
index 0000000..f2f0533
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.web;
+
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+import io.netty.handler.codec.http.DefaultHttpResponse;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.util.ReferenceCountUtil;
+
+import org.apache.commons.logging.Log;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
+import org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction;
+
+/**
+ * Netty handler that integrates with the {@link RestCsrfPreventionFilter}. If
+ * the filter determines that the request is allowed, then this handler forwards
+ * the request to the next handler in the Netty pipeline. Otherwise, this
+ * handler drops the request and immediately sends an HTTP 400 response.
+ */
+@InterfaceAudience.Private
+final class RestCsrfPreventionFilterHandler
+ extends SimpleChannelInboundHandler<HttpRequest> {
+
+ private static final Log LOG = DatanodeHttpServer.LOG;
+
+ private final RestCsrfPreventionFilter restCsrfPreventionFilter;
+
+ /**
+ * Creates a new RestCsrfPreventionFilterHandler. There will be a new
+ * instance created for each new Netty channel/pipeline serving a new request.
+ * To prevent the cost of repeated initialization of the filter, this
+ * constructor requires the caller to pass in a pre-built, fully initialized
+ * filter instance. The filter is stateless after initialization, so it can
+ * be shared across multiple Netty channels/pipelines.
+ *
+ * @param restCsrfPreventionFilter initialized filter
+ */
+ public RestCsrfPreventionFilterHandler(
+ RestCsrfPreventionFilter restCsrfPreventionFilter) {
+ this.restCsrfPreventionFilter = restCsrfPreventionFilter;
+ }
+
+ @Override
+ protected void channelRead0(final ChannelHandlerContext ctx,
+ final HttpRequest req) throws Exception {
+ restCsrfPreventionFilter.handleHttpInteraction(new NettyHttpInteraction(
+ ctx, req));
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ LOG.error("Exception in " + this.getClass().getSimpleName(), cause);
+ sendResponseAndClose(ctx,
+ new DefaultHttpResponse(HTTP_1_1, INTERNAL_SERVER_ERROR));
+ }
+
+ /**
+ * Finish handling this pipeline by writing a response with the
+ * "Connection: close" header, flushing, and scheduling a close of the
+ * connection.
+ *
+ * @param ctx context to receive the response
+ * @param resp response to send
+ */
+ private static void sendResponseAndClose(ChannelHandlerContext ctx,
+ DefaultHttpResponse resp) {
+ resp.headers().set(CONNECTION, CLOSE);
+ ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE);
+ }
+
+ /**
+ * {@link HttpInteraction} implementation for use in a Netty pipeline.
+ */
+ private static final class NettyHttpInteraction implements HttpInteraction {
+
+ private final ChannelHandlerContext ctx;
+ private final HttpRequest req;
+
+ /**
+ * Creates a new NettyHttpInteraction.
+ *
+ * @param ctx context to receive the response
+ * @param req request to process
+ */
+ public NettyHttpInteraction(ChannelHandlerContext ctx, HttpRequest req) {
+ this.ctx = ctx;
+ this.req = req;
+ }
+
+ @Override
+ public String getHeader(String header) {
+ return req.headers().get(header);
+ }
+
+ @Override
+ public String getMethod() {
+ return req.method().name();
+ }
+
+ @Override
+ public void proceed() {
+ ReferenceCountUtil.retain(req);
+ ctx.fireChannelRead(req);
+ }
+
+ @Override
+ public void sendError(int code, String message) {
+ HttpResponseStatus status = new HttpResponseStatus(code, message);
+ sendResponseAndClose(ctx, new DefaultHttpResponse(HTTP_1_1, status));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index 55cf00f..84229e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_KEY;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -45,6 +47,7 @@ import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
/**
* Encapsulates the HTTP server started by the NameNode.
@@ -90,6 +93,16 @@ public class NameNodeHttpServer {
HttpServer2.LOG.info("Added filter '" + name + "' (class=" + className
+ ")");
+ // add REST CSRF prevention filter
+ if (conf.getBoolean(DFS_WEBHDFS_REST_CSRF_ENABLED_KEY,
+ DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT)) {
+ Map<String, String> restCsrfParams = RestCsrfPreventionFilter
+ .getFilterParams(conf, "dfs.webhdfs.rest-csrf.");
+ String restCsrfClassName = RestCsrfPreventionFilter.class.getName();
+ HttpServer2.defineFilter(httpServer.getWebAppContext(), restCsrfClassName,
+ restCsrfClassName, restCsrfParams, new String[] {pathSpec});
+ }
+
// add webhdfs packages
httpServer.addJerseyResourcePackage(NamenodeWebHdfsMethods.class
.getPackage().getName() + ";" + Param.class.getPackage().getName(),
@@ -297,4 +310,4 @@ public class NameNodeHttpServer {
public static HAServiceProtocol.HAServiceState getNameNodeStateFromContext(ServletContext context) {
return getNameNodeFromContext(context).getServiceState();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4889bc3..0f5130a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2762,4 +2762,56 @@
</description>
</property>
+<property>
+ <name>dfs.webhdfs.rest-csrf.enabled</name>
+ <value>false</value>
+ <description>
+ If true, then enables WebHDFS protection against cross-site request forgery
+ (CSRF). The WebHDFS client also uses this property to determine whether or
+ not it needs to send the custom CSRF prevention header in its HTTP requests.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.rest-csrf.custom-header</name>
+ <value>X-XSRF-HEADER</value>
+ <description>
+ The name of a custom header that HTTP requests must send when protection
+ against cross-site request forgery (CSRF) is enabled for WebHDFS by setting
+ dfs.webhdfs.rest-csrf.enabled to true. The WebHDFS client also uses this
+ property to determine whether or not it needs to send the custom CSRF
+ prevention header in its HTTP requests.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.rest-csrf.methods-to-ignore</name>
+ <value>GET,OPTIONS,HEAD,TRACE</value>
+ <description>
+ A comma-separated list of HTTP methods that do not require HTTP requests to
+ include a custom header when protection against cross-site request forgery
+ (CSRF) is enabled for WebHDFS by setting dfs.webhdfs.rest-csrf.enabled to
+ true. The WebHDFS client also uses this property to determine whether or
+ not it needs to send the custom CSRF prevention header in its HTTP requests.
+ </description>
+</property>
+
+<property>
+ <name>dfs.webhdfs.rest-csrf.browser-useragents-regex</name>
+ <value>^Mozilla.*,^Opera.*</value>
+ <description>
+ A comma-separated list of regular expressions used to match against an HTTP
+ request's User-Agent header when protection against cross-site request
+ forgery (CSRF) is enabled for WebHDFS by setting
+ dfs.webhdfs.reset-csrf.enabled to true. If the incoming User-Agent matches
+ any of these regular expressions, then the request is considered to be sent
+ by a browser, and therefore CSRF prevention is enforced. If the request's
+ User-Agent does not match any of these regular expressions, then the request
+ is considered to be sent by something other than a browser, such as scripted
+ automation. In this case, CSRF is not a potential attack vector, so
+ the prevention is not enforced. This helps achieve backwards-compatibility
+ with existing automation that has not been updated to send the CSRF
+ prevention header.
+ </description>
+</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index b68cac2..e66e1e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -269,6 +269,7 @@
</script><script type="text/javascript" src="/static/dust-helpers-1.1.1.min.js">
</script><script type="text/javascript" src="/static/dfs-dust.js">
</script><script type="text/javascript" src="/static/json-bignum.js">
+ </script><script type="text/javascript" src="/static/rest-csrf.js">
</script><script type="text/javascript" src="explorer.js">
</script><script type="text/javascript" src="/static/moment.min.js">
</script>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/rest-csrf.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/rest-csrf.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/rest-csrf.js
new file mode 100644
index 0000000..973a8e8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/rest-csrf.js
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+"use strict";
+
+// Initializes client-side handling of cross-site request forgery (CSRF)
+// protection by figuring out the custom HTTP headers that need to be sent in
+// requests and which HTTP methods are ignored because they do not require CSRF
+// protection.
+(function() {
+ var restCsrfCustomHeader = null;
+ var restCsrfMethodsToIgnore = null;
+
+ $.ajax({'url': '/conf', 'dataType': 'xml', 'async': false}).done(
+ function(data) {
+ function getBooleanValue(element) {
+ return ($(element).find('value').text().trim().toLowerCase() === 'true')
+ }
+
+ function getTrimmedStringValue(element) {
+ return $(element).find('value').text().trim();
+ }
+
+ function getTrimmedStringArrayValue(element) {
+ var str = $(element).find('value').text().trim();
+ var array = [];
+ if (str) {
+ var splitStr = str.split(',');
+ for (var i = 0; i < splitStr.length; i++) {
+ array.push(splitStr[i].trim());
+ }
+ }
+ return array;
+ }
+
+ // Get all relevant configuration properties.
+ var $xml = $(data);
+ var csrfEnabled = false;
+ var header = null;
+ var methods = [];
+ $xml.find('property').each(function(idx, element) {
+ var name = $(element).find('name').text();
+ if (name === 'dfs.webhdfs.rest-csrf.enabled') {
+ csrfEnabled = getBooleanValue(element);
+ } else if (name === 'dfs.webhdfs.rest-csrf.custom-header') {
+ header = getTrimmedStringValue(element);
+ } else if (name === 'dfs.webhdfs.rest-csrf.methods-to-ignore') {
+ methods = getTrimmedStringArrayValue(element);
+ }
+ });
+
+ // If enabled, set up all subsequent AJAX calls with a pre-send callback
+ // that adds the custom headers if necessary.
+ if (csrfEnabled) {
+ restCsrfCustomHeader = header;
+ restCsrfMethodsToIgnore = {};
+ methods.map(function(method) { restCsrfMethodsToIgnore[method] = true; });
+ $.ajaxSetup({
+ beforeSend: addRestCsrfCustomHeader
+ });
+ }
+ });
+
+ // Adds custom headers to request if necessary. This is done only for WebHDFS
+ // URLs, and only if it's not an ignored method.
+ function addRestCsrfCustomHeader(xhr, settings) {
+ if (settings.url == null || !settings.url.startsWith('/webhdfs/')) {
+ return;
+ }
+ var method = settings.type;
+ if (restCsrfCustomHeader != null && !restCsrfMethodsToIgnore[method]) {
+ // The value of the header is unimportant. Only its presence matters.
+ xhr.setRequestHeader(restCsrfCustomHeader, '""');
+ }
+ }
+})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 2b7a493..473ad27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -23,6 +23,7 @@ WebHDFS REST API
* [HDFS Configuration Options](#HDFS_Configuration_Options)
* [Authentication](#Authentication)
* [Proxy Users](#Proxy_Users)
+ * [Cross-Site Request Forgery Prevention](#Cross-Site_Request_Forgery_Prevention)
* [File and Directory Operations](#File_and_Directory_Operations)
* [Create and Write to a File](#Create_and_Write_to_a_File)
* [Append to a File](#Append_to_a_File)
@@ -263,6 +264,41 @@ When the proxy user feature is enabled, a proxy user *P* may submit a request on
curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?delegation=<TOKEN>&op=..."
+Cross-Site Request Forgery Prevention
+-------------------------------------
+
+WebHDFS supports an optional, configurable mechanism for cross-site request
+forgery (CSRF) prevention. When enabled, WebHDFS HTTP requests to the NameNode
+or DataNode must include a custom HTTP header. Configuration properties allow
+adjusting which specific HTTP methods are protected and the name of the HTTP
+header. The value sent in the header is not relevant. Only the presence of a
+header by that name is required.
+
+Enabling CSRF prevention also sets up the `WebHdfsFileSystem` class to send the
+required header. This ensures that CLI commands like
+[`hdfs dfs`](./HDFSCommands.html#dfs) and
+[`hadoop distcp`](../../hadoop-distcp/DistCp.html) continue to work correctly
+when used with `webhdfs:` URIs.
+
+Enabling CSRF prevention also sets up the NameNode web UI to send the required
+header. After enabling CSRF prevention and restarting the NameNode, existing
+users of the NameNode web UI need to refresh the browser to reload the page and
+find the new configuration.
+
+The following properties control CSRF prevention.
+
+| Property | Description | Default Value |
+|:---- |:---- |:----
+| `dfs.webhdfs.rest-csrf.enabled` | If true, then enables WebHDFS protection against cross-site request forgery (CSRF). The WebHDFS client also uses this property to determine whether or not it needs to send the custom CSRF prevention header in its HTTP requests. | `false` |
+| `dfs.webhdfs.rest-csrf.custom-header` | The name of a custom header that HTTP requests must send when protection against cross-site request forgery (CSRF) is enabled for WebHDFS by setting dfs.webhdfs.rest-csrf.enabled to true. The WebHDFS client also uses this property to determine whether or not it needs to send the custom CSRF prevention header in its HTTP requests. | `X-XSRF-HEADER` |
+| `dfs.webhdfs.rest-csrf.methods-to-ignore` | A comma-separated list of HTTP methods that do not require HTTP requests to include a custom header when protection against cross-site request forgery (CSRF) is enabled for WebHDFS by setting dfs.webhdfs.rest-csrf.enabled to true. The WebHDFS client also uses this property to determine whether or not it needs to send the custom CSRF prevention header in its HTTP requests. | `GET,OPTIONS,HEAD,TRACE` |
+| `dfs.webhdfs.rest-csrf.browser-useragents-regex` | A comma-separated list of regular expressions used to match against an HTTP request's User-Agent header when protection against cross-site request forgery (CSRF) is enabled for WebHDFS by setting dfs.webhdfs.reset-csrf.enabled to true. If the incoming User-Agent matches any of these regular expressions, then the request is considered to be sent by a browser, and therefore CSRF prevention is enforced. If the request's User-Agent does not match any of these regular expressions, then the request is considered to be sent by something other than a browser, such as scripted automation. In this case, CSRF is not a potential attack vector, so the prevention is not enforced. This helps achieve backwards-compatibility with existing automation that has not been updated to send the CSRF prevention header. | `^Mozilla.*,^Opera.*` |
+
+The following is an example `curl` call that uses the `-H` option to include the
+custom header in the request.
+
+ curl -i -L -X PUT -H 'X-XSRF-HEADER: ""' 'http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE'
+
File and Directory Operations
-----------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d1889a6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithRestCsrfPreventionFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithRestCsrfPreventionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithRestCsrfPreventionFilter.java
new file mode 100644
index 0000000..d5f4a05
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithRestCsrfPreventionFilter.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_BROWSER_USERAGENTS_REGEX_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_KEY;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Tests use of the cross-site-request forgery (CSRF) prevention filter with
+ * WebHDFS. This is a parameterized test that covers various combinations of
+ * CSRF protection enabled or disabled at the NameNode, the DataNode and the
+ * WebHDFS client. If the server is configured with CSRF prevention, but the
+ * client is not, then protected operations are expected to fail.
+ */
+@RunWith(Parameterized.class)
+public class TestWebHdfsWithRestCsrfPreventionFilter {
+
+ private static final Path FILE = new Path("/file");
+
+ private final boolean nnRestCsrf;
+ private final boolean dnRestCsrf;
+ private final boolean clientRestCsrf;
+
+ private MiniDFSCluster cluster;
+ private FileSystem fs, webhdfs;
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ public TestWebHdfsWithRestCsrfPreventionFilter(boolean nnRestCsrf,
+ boolean dnRestCsrf, boolean clientRestCsrf) {
+ this.nnRestCsrf = nnRestCsrf;
+ this.dnRestCsrf = dnRestCsrf;
+ this.clientRestCsrf = clientRestCsrf;
+ }
+
+ @Parameters
+ public static Iterable<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { false, false, false },
+ { true, true, true },
+ { true, true, false },
+ { true, false, true },
+ { true, false, false },
+ { false, true, true },
+ { false, true, false },
+ { false, false, true }});
+ }
+
+ @Before
+ public void before() throws Exception {
+ Configuration nnConf = new Configuration();
+ nnConf.setBoolean(DFS_WEBHDFS_REST_CSRF_ENABLED_KEY, nnRestCsrf);
+ // Set configuration to treat anything as a browser, so that CSRF prevention
+ // checks actually get enforced.
+ nnConf.set(DFS_WEBHDFS_REST_CSRF_BROWSER_USERAGENTS_REGEX_KEY, ".*");
+ cluster = new MiniDFSCluster.Builder(nnConf).numDataNodes(0).build();
+
+ Configuration dnConf = new Configuration(nnConf);
+ dnConf.setBoolean(DFS_WEBHDFS_REST_CSRF_ENABLED_KEY, dnRestCsrf);
+ cluster.startDataNodes(dnConf, 1, true, null, null, null, null, false);
+
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+
+ Configuration clientConf = new Configuration();
+ clientConf.setBoolean(DFS_WEBHDFS_REST_CSRF_ENABLED_KEY, clientRestCsrf);
+ InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
+ webhdfs = FileSystem.get(URI.create("webhdfs://" +
+ NetUtils.getHostPortString(addr)), clientConf);
+ }
+
+ @After
+ public void after() {
+ IOUtils.closeStream(webhdfs);
+ IOUtils.closeStream(fs);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testCreate() throws Exception {
+ // create is a HTTP PUT that redirects from NameNode to DataNode, so we
+ // expect CSRF prevention on either server to block an unconfigured client.
+ if ((nnRestCsrf || dnRestCsrf) && !clientRestCsrf) {
+ expectException();
+ }
+ assertTrue(webhdfs.createNewFile(FILE));
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ DFSTestUtil.createFile(fs, FILE, 1024, (short)1, 0L);
+ // delete is an HTTP DELETE that executes solely within the NameNode as a
+ // metadata operation, so we expect CSRF prevention configured on the
+ // NameNode to block an unconfigured client.
+ if (nnRestCsrf && !clientRestCsrf) {
+ expectException();
+ }
+ assertTrue(webhdfs.delete(FILE, false));
+ }
+
+ @Test
+ public void testGetFileStatus() throws Exception {
+ // getFileStatus is an HTTP GET, not subject to CSRF prevention, so we
+ // expect it to succeed always, regardless of CSRF configuration.
+ assertNotNull(webhdfs.getFileStatus(new Path("/")));
+ }
+
+ @Test
+ public void testTruncate() throws Exception {
+ DFSTestUtil.createFile(fs, FILE, 1024, (short)1, 0L);
+ // truncate is an HTTP POST that executes solely within the NameNode as a
+ // metadata operation, so we expect CSRF prevention configured on the
+ // NameNode to block an unconfigured client.
+ if (nnRestCsrf && !clientRestCsrf) {
+ expectException();
+ }
+ assertTrue(webhdfs.truncate(FILE, 0L));
+ }
+
+ private void expectException() {
+ exception.expect(IOException.class);
+ exception.expectMessage("Missing Required Header");
+ }
+}
[15/50] [abbrv] hadoop git commit: HDFS-9608. Disk IO imbalance in
HDFS with heterogeneous storages. Contributed by Wei Zhou.
Posted by ar...@apache.org.
HDFS-9608. Disk IO imbalance in HDFS with heterogeneous storages. Contributed by Wei Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a23dc68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a23dc68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a23dc68
Branch: refs/heads/HDFS-1312
Commit: 3a23dc683c058d3a5262ae9dca2d1c8c588a6a3e
Parents: 77f7ca3
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Feb 17 11:29:10 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Feb 17 11:29:58 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../AvailableSpaceVolumeChoosingPolicy.java | 35 ++++++++++--
.../RoundRobinVolumeChoosingPolicy.java | 53 +++++++++++++-----
.../TestRoundRobinVolumeChoosingPolicy.java | 56 ++++++++++++++++++++
4 files changed, 131 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a23dc68/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5ca7f8a..36bb60d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1021,6 +1021,9 @@ Release 2.9.0 - UNRELEASED
HDFS-9691. TestBlockManagerSafeMode#testCheckSafeMode fails intermittently.
(Mingliang Liu via aajisaka)
+ HDFS-9608. Disk IO imbalance in HDFS with heterogeneous storages.
+ (Wei Zhou via wang)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a23dc68/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
index d0d36ba..39d9547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
@@ -31,6 +31,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
/**
@@ -39,11 +40,15 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
* new replica allocation. By default this policy prefers assigning replicas to
* those volumes with more available free space, so as to over time balance the
* available space of all the volumes within a DN.
+ * Use fine-grained locks to enable choosing volumes of different storage
+ * types concurrently.
*/
public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
implements VolumeChoosingPolicy<V>, Configurable {
private static final Log LOG = LogFactory.getLog(AvailableSpaceVolumeChoosingPolicy.class);
+
+ private Object[] syncLocks;
private final Random random;
@@ -52,14 +57,24 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
AvailableSpaceVolumeChoosingPolicy(Random random) {
this.random = random;
+ initLocks();
}
public AvailableSpaceVolumeChoosingPolicy() {
this(new Random());
+ initLocks();
+ }
+
+ private void initLocks() {
+ int numStorageTypes = StorageType.values().length;
+ syncLocks = new Object[numStorageTypes];
+ for (int i = 0; i < numStorageTypes; i++) {
+ syncLocks[i] = new Object();
+ }
}
@Override
- public synchronized void setConf(Configuration conf) {
+ public void setConf(Configuration conf) {
balancedSpaceThreshold = conf.getLong(
DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_KEY,
DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT);
@@ -85,7 +100,7 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
}
@Override
- public synchronized Configuration getConf() {
+ public Configuration getConf() {
// Nothing to do. Only added to fulfill the Configurable contract.
return null;
}
@@ -98,12 +113,24 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
new RoundRobinVolumeChoosingPolicy<V>();
@Override
- public synchronized V chooseVolume(List<V> volumes,
+ public V chooseVolume(List<V> volumes,
long replicaSize) throws IOException {
if (volumes.size() < 1) {
throw new DiskOutOfSpaceException("No more available volumes");
}
-
+ // As all the items in volumes are with the same storage type,
+ // so only need to get the storage type index of the first item in volumes
+ StorageType storageType = volumes.get(0).getStorageType();
+ int index = storageType != null ?
+ storageType.ordinal() : StorageType.DEFAULT.ordinal();
+
+ synchronized (syncLocks[index]) {
+ return doChooseVolume(volumes, replicaSize);
+ }
+ }
+
+ private V doChooseVolume(final List<V> volumes,
+ long replicaSize) throws IOException {
AvailableSpaceVolumeList volumesWithSpaces =
new AvailableSpaceVolumeList(volumes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a23dc68/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
index 19452d8..9474b92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
@@ -22,46 +22,75 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
/**
- * Choose volumes in round-robin order.
+ * Choose volumes with the same storage type in round-robin order.
+ * Use fine-grained locks to synchronize volume choosing.
*/
public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
implements VolumeChoosingPolicy<V> {
public static final Log LOG = LogFactory.getLog(RoundRobinVolumeChoosingPolicy.class);
- private int curVolume = 0;
+ // curVolumes stores the RR counters of each storage type.
+ // The ordinal of storage type in org.apache.hadoop.fs.StorageType
+ // is used as the index to get data from the array.
+ private int[] curVolumes;
+ // syncLocks stores the locks for each storage type.
+ private Object[] syncLocks;
+
+ public RoundRobinVolumeChoosingPolicy() {
+ int numStorageTypes = StorageType.values().length;
+ curVolumes = new int[numStorageTypes];
+ syncLocks = new Object[numStorageTypes];
+ for (int i = 0; i < numStorageTypes; i++) {
+ syncLocks[i] = new Object();
+ }
+ }
@Override
- public synchronized V chooseVolume(final List<V> volumes, long blockSize)
+ public V chooseVolume(final List<V> volumes, long blockSize)
throws IOException {
- if(volumes.size() < 1) {
+ if (volumes.size() < 1) {
throw new DiskOutOfSpaceException("No more available volumes");
}
-
+
+ // As all the items in volumes are with the same storage type,
+ // so only need to get the storage type index of the first item in volumes
+ StorageType storageType = volumes.get(0).getStorageType();
+ int index = storageType != null ?
+ storageType.ordinal() : StorageType.DEFAULT.ordinal();
+
+ synchronized (syncLocks[index]) {
+ return chooseVolume(index, volumes, blockSize);
+ }
+ }
+
+ private V chooseVolume(final int curVolumeIndex, final List<V> volumes,
+ long blockSize) throws IOException {
// since volumes could've been removed because of the failure
// make sure we are not out of bounds
- if(curVolume >= volumes.size()) {
- curVolume = 0;
- }
-
+ int curVolume = curVolumes[curVolumeIndex] < volumes.size()
+ ? curVolumes[curVolumeIndex] : 0;
+
int startVolume = curVolume;
long maxAvailable = 0;
-
+
while (true) {
final V volume = volumes.get(curVolume);
curVolume = (curVolume + 1) % volumes.size();
long availableVolumeSize = volume.getAvailable();
if (availableVolumeSize > blockSize) {
+ curVolumes[curVolumeIndex] = curVolume;
return volume;
}
-
+
if (availableVolumeSize > maxAvailable) {
maxAvailable = availableVolumeSize;
}
-
+
if (curVolume == startVolume) {
throw new DiskOutOfSpaceException("Out of space: "
+ "The volume with the most available space (=" + maxAvailable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a23dc68/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
index 9818a01..9b3047f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Assert;
@@ -102,4 +103,59 @@ public class TestRoundRobinVolumeChoosingPolicy {
}
}
+ // Test Round-Robin choosing algorithm with heterogeneous storage.
+ @Test
+ public void testRRPolicyWithStorageTypes() throws Exception {
+ final RoundRobinVolumeChoosingPolicy<FsVolumeSpi> policy
+ = new RoundRobinVolumeChoosingPolicy<FsVolumeSpi>();
+ testRRPolicyWithStorageTypes(policy);
+ }
+
+ public static void testRRPolicyWithStorageTypes(
+ VolumeChoosingPolicy<FsVolumeSpi> policy) throws Exception {
+ final List<FsVolumeSpi> diskVolumes = new ArrayList<FsVolumeSpi>();
+ final List<FsVolumeSpi> ssdVolumes = new ArrayList<FsVolumeSpi>();
+
+ // Add two DISK volumes to diskVolumes
+ diskVolumes.add(Mockito.mock(FsVolumeSpi.class));
+ Mockito.when(diskVolumes.get(0).getStorageType())
+ .thenReturn(StorageType.DISK);
+ Mockito.when(diskVolumes.get(0).getAvailable()).thenReturn(100L);
+ diskVolumes.add(Mockito.mock(FsVolumeSpi.class));
+ Mockito.when(diskVolumes.get(1).getStorageType())
+ .thenReturn(StorageType.DISK);
+ Mockito.when(diskVolumes.get(1).getAvailable()).thenReturn(100L);
+
+ // Add two SSD volumes to ssdVolumes
+ ssdVolumes.add(Mockito.mock(FsVolumeSpi.class));
+ Mockito.when(ssdVolumes.get(0).getStorageType())
+ .thenReturn(StorageType.SSD);
+ Mockito.when(ssdVolumes.get(0).getAvailable()).thenReturn(200L);
+ ssdVolumes.add(Mockito.mock(FsVolumeSpi.class));
+ Mockito.when(ssdVolumes.get(1).getStorageType())
+ .thenReturn(StorageType.SSD);
+ Mockito.when(ssdVolumes.get(1).getAvailable()).thenReturn(100L);
+
+ Assert.assertEquals(diskVolumes.get(0),
+ policy.chooseVolume(diskVolumes, 0));
+ // Independent Round-Robin for different storage type
+ Assert.assertEquals(ssdVolumes.get(0),
+ policy.chooseVolume(ssdVolumes, 0));
+ // Take block size into consideration
+ Assert.assertEquals(ssdVolumes.get(0),
+ policy.chooseVolume(ssdVolumes, 150L));
+
+ Assert.assertEquals(diskVolumes.get(1),
+ policy.chooseVolume(diskVolumes, 0));
+ Assert.assertEquals(diskVolumes.get(0),
+ policy.chooseVolume(diskVolumes, 50L));
+
+ try {
+ policy.chooseVolume(diskVolumes, 200L);
+ Assert.fail("Should throw an DiskOutOfSpaceException before this!");
+ } catch (DiskOutOfSpaceException e) {
+ // Pass.
+ }
+ }
+
}
[23/50] [abbrv] hadoop git commit: MAPREDUCE-6634. Log uncaught
exceptions/errors in various thread pools in mapreduce. Contributed by
Sidharta Seethana.
Posted by ar...@apache.org.
MAPREDUCE-6634. Log uncaught exceptions/errors in various thread pools in mapreduce. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2440671a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2440671a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2440671a
Branch: refs/heads/HDFS-1312
Commit: 2440671a117f165dcda5056404bc898df3c50803
Parents: c1afac3
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Feb 18 14:15:08 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Feb 18 14:18:36 2016 +0530
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapred/LocalContainerLauncher.java | 4 +--
.../v2/app/commit/CommitterEventHandler.java | 3 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 3 +-
.../v2/app/launcher/ContainerLauncherImpl.java | 3 +-
.../mapred/LocalDistributedCacheManager.java | 5 ++--
.../apache/hadoop/mapred/LocalJobRunner.java | 8 +++--
.../hadoop/mapred/LocatedFileStatusFetcher.java | 4 +--
.../java/org/apache/hadoop/mapred/TaskLog.java | 31 ++++++++++----------
.../mapred/lib/MultithreadedMapRunner.java | 4 ++-
.../lib/output/TestFileOutputCommitter.java | 4 +--
.../mapreduce/v2/hs/HistoryFileManager.java | 6 ++--
.../hadoop/mapreduce/v2/hs/JobHistory.java | 3 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 6 ++--
.../org/apache/hadoop/examples/pi/Util.java | 5 ++--
15 files changed, 53 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 0b8c818..da28bc6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -310,6 +310,9 @@ Release 2.9.0 - UNRELEASED
MAPREDUCE-6431. JobClient should be an AutoClosable (haibochen via rkanter)
+ MAPREDUCE-6634. Log uncaught exceptions/errors in various thread pools in
+ mapreduce. (Sidharta Seethana via vvasudev)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 1a0d5fb..da118c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -26,7 +26,6 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
@@ -60,6 +59,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -138,7 +138,7 @@ public class LocalContainerLauncher extends AbstractService implements
// make it a daemon thread so that the process can exit even if the task is
// not interruptible
taskRunner =
- Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().
+ HadoopExecutors.newSingleThreadExecutor(new ThreadFactoryBuilder().
setDaemon(true).setNameFormat("uber-SubtaskRunner").build());
// create and start an event handling thread
eventHandler = new Thread(new EventHandler(), "uber-EventHandler");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
index b53955f..0b1be70 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -133,7 +134,7 @@ public class CommitterEventHandler extends AbstractService
tfBuilder.setThreadFactory(backingTf);
}
ThreadFactory tf = tfBuilder.build();
- launcherPool = new ThreadPoolExecutor(5, 5, 1,
+ launcherPool = new HadoopThreadPoolExecutor(5, 5, 1,
TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf);
eventHandlingThread = new Thread(new Runnable() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 5ed0762..c8c5ce9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -116,6 +116,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.concurrent.HadoopScheduledThreadPoolExecutor;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -698,7 +699,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
.setNameFormat("Job Fail Wait Timeout Monitor #%d")
.setDaemon(true)
.build();
- this.executor = new ScheduledThreadPoolExecutor(1, threadFactory);
+ this.executor = new HadoopScheduledThreadPoolExecutor(1, threadFactory);
// This "this leak" is okay because the retained pointer is in an
// instance variable.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
index a7e966c..189e2ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
@@ -266,7 +267,7 @@ public class ContainerLauncherImpl extends AbstractService implements
"ContainerLauncher #%d").setDaemon(true).build();
// Start with a default core-pool size of 10 and change it dynamically.
- launcherPool = new ThreadPoolExecutor(initialPoolSize,
+ launcherPool = new HadoopThreadPoolExecutor(initialPoolSize,
Integer.MAX_VALUE, 1, TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
index 8606ede..3b87197 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
@@ -35,7 +35,6 @@ import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicLong;
@@ -43,7 +42,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
@@ -53,6 +51,7 @@ import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -121,7 +120,7 @@ class LocalDistributedCacheManager {
ThreadFactory tf = new ThreadFactoryBuilder()
.setNameFormat("LocalDistributedCacheManager Downloader #%d")
.build();
- exec = Executors.newCachedThreadPool(tf);
+ exec = HadoopExecutors.newCachedThreadPool(tf);
Path destPath = localDirAllocator.getLocalPathForWrite(".", conf);
Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap();
for (LocalResource resource : localResources.values()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 45d3cc5..37c147d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -32,7 +32,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -74,6 +73,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
/** Implements MapReduce locally, in-process, for debugging. */
@InterfaceAudience.Private
@@ -428,7 +428,8 @@ public class LocalJobRunner implements ClientProtocol {
ThreadFactory tf = new ThreadFactoryBuilder()
.setNameFormat("LocalJobRunner Map Task Executor #%d")
.build();
- ExecutorService executor = Executors.newFixedThreadPool(maxMapThreads, tf);
+ ExecutorService executor = HadoopExecutors.newFixedThreadPool(
+ maxMapThreads, tf);
return executor;
}
@@ -454,7 +455,8 @@ public class LocalJobRunner implements ClientProtocol {
LOG.debug("Reduce tasks to process: " + this.numReduceTasks);
// Create a new executor service to drain the work queue.
- ExecutorService executor = Executors.newFixedThreadPool(maxReduceThreads);
+ ExecutorService executor = HadoopExecutors.newFixedThreadPool(
+ maxReduceThreads);
return executor;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
index 87114ad..a039bc9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LocatedFileStatusFetcher.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
@@ -47,6 +46,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
/**
* Utility class to fetch block locations for specified Input paths using a
@@ -92,7 +92,7 @@ public class LocatedFileStatusFetcher {
IOException {
int numThreads = conf.getInt(FileInputFormat.LIST_STATUS_NUM_THREADS,
FileInputFormat.DEFAULT_LIST_STATUS_NUM_THREADS);
- rawExec = Executors.newFixedThreadPool(
+ rawExec = HadoopExecutors.newFixedThreadPool(
numThreads,
new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("GetFileInfo #%d").build());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
index e07b5be..bf838c2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.mapreduce.util.ProcessTree;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.log4j.Appender;
import org.apache.log4j.LogManager;
@@ -327,22 +328,22 @@ public class TaskLog {
public static ScheduledExecutorService createLogSyncer() {
final ScheduledExecutorService scheduler =
- Executors.newSingleThreadScheduledExecutor(
- new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- final Thread t = Executors.defaultThreadFactory().newThread(r);
- t.setDaemon(true);
- t.setName("Thread for syncLogs");
- return t;
- }
- });
+ HadoopExecutors.newSingleThreadScheduledExecutor(
+ new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ final Thread t = Executors.defaultThreadFactory().newThread(r);
+ t.setDaemon(true);
+ t.setName("Thread for syncLogs");
+ return t;
+ }
+ });
ShutdownHookManager.get().addShutdownHook(new Runnable() {
- @Override
- public void run() {
- TaskLog.syncLogsShutdown(scheduler);
- }
- }, 50);
+ @Override
+ public void run() {
+ TaskLog.syncLogsShutdown(scheduler);
+ }
+ }, 50);
scheduler.scheduleWithFixedDelay(
new Runnable() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultithreadedMapRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultithreadedMapRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultithreadedMapRunner.java
index 98d794b..05339bc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultithreadedMapRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultithreadedMapRunner.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.SkipBadRecords;
import org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
import java.io.IOException;
import java.util.concurrent.*;
@@ -84,7 +85,8 @@ public class MultithreadedMapRunner<K1, V1, K2, V2>
// Creating a threadpool of the configured size to execute the Mapper
// map method in parallel.
- executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads,
+ executorService = new HadoopThreadPoolExecutor(numberOfThreads,
+ numberOfThreads,
0L, TimeUnit.MILLISECONDS,
new BlockingArrayQueue
(numberOfThreads));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
index eba513b..20d8ab5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
@@ -25,10 +25,10 @@ import java.io.IOException;
import java.net.URI;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import junit.framework.TestCase;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -696,7 +696,7 @@ public class TestFileOutputCommitter extends TestCase {
};
}
- final ExecutorService executor = Executors.newFixedThreadPool(2);
+ final ExecutorService executor = HadoopExecutors.newFixedThreadPool(2);
try {
for (int i = 0; i < taCtx.length; i++) {
final int taskIdx = i;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6be0d27..677d5c2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.ShutdownThreadsHelper;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
@@ -554,8 +555,9 @@ public class HistoryFileManager extends AbstractService {
JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
"MoveIntermediateToDone Thread #%d").build();
- moveToDoneExecutor = new ThreadPoolExecutor(numMoveThreads, numMoveThreads,
- 1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf);
+ moveToDoneExecutor = new HadoopThreadPoolExecutor(numMoveThreads,
+ numMoveThreads, 1, TimeUnit.HOURS,
+ new LinkedBlockingQueue<Runnable>(), tf);
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
index 41bc90a..45075c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.concurrent.HadoopScheduledThreadPoolExecutor;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -126,7 +127,7 @@ public class JobHistory extends AbstractService implements HistoryContext {
((Service) storage).start();
}
- scheduledExecutor = new ScheduledThreadPoolExecutor(2,
+ scheduledExecutor = new HadoopScheduledThreadPoolExecutor(2,
new ThreadFactoryBuilder().setNameFormat("Log Scanner/Cleaner #%d")
.build());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 2fb7811..0d6e900 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -46,7 +46,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -81,6 +80,7 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
@@ -475,8 +475,8 @@ public class ShuffleHandler extends AuxiliaryService {
.build();
selector = new NioServerSocketChannelFactory(
- Executors.newCachedThreadPool(bossFactory),
- Executors.newCachedThreadPool(workerFactory),
+ HadoopExecutors.newCachedThreadPool(bossFactory),
+ HadoopExecutors.newCachedThreadPool(workerFactory),
maxShuffleThreads);
super.serviceInit(new Configuration(conf));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2440671a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java
index 8afc1bd..e74c091 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java
@@ -35,7 +35,6 @@ import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
@@ -48,6 +47,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Charsets;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
/** Utility methods */
public class Util {
@@ -157,7 +157,8 @@ public class Util {
/** Execute the callables by a number of threads */
public static <T, E extends Callable<T>> void execute(int nThreads, List<E> callables
) throws InterruptedException, ExecutionException {
- final ExecutorService executor = Executors.newFixedThreadPool(nThreads);
+ final ExecutorService executor = HadoopExecutors.newFixedThreadPool(
+ nThreads);
final List<Future<T>> futures = executor.invokeAll(callables);
for(Future<T> f : futures)
f.get();
[19/50] [abbrv] hadoop git commit: YARN-4689. Fix up CHANGES.txt
Posted by ar...@apache.org.
YARN-4689. Fix up CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c248ea4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c248ea4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c248ea4
Branch: refs/heads/HDFS-1312
Commit: 1c248ea4a8442bccc78a8ce4539ce1f4b6d6b0ee
Parents: 2ab4c47
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Feb 17 18:18:46 2016 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Feb 17 18:18:46 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c248ea4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f7c69a4..4471912 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -34,8 +34,6 @@ Trunk - Unreleased
YARN-3261. rewrite resourcemanager restart doc to remove roadmap bits (Gururaj Shetty via aw)
- YARN-4689. FairScheduler: Cleanup preemptContainer to be more readable.
- (Kai Sasaki via kasha)
OPTIMIZATIONS
@@ -134,6 +132,10 @@ Release 2.9.0 - UNRELEASED
YARN-4655. Log uncaught exceptions/errors in various thread pools in YARN.
(Sidharta Seethana via vvasudev)
+ YARN-4689. FairScheduler: Cleanup preemptContainer to be more readable.
+ (Kai Sasaki via kasha)
+
+
OPTIMIZATIONS
BUG FIXES
[36/50] [abbrv] hadoop git commit: HDFS-9764. DistCp doesn't print
value for several arguments including -numListstatusThreads. (Wei-Chiu Chuang
via Yongjun Zhang)
Posted by ar...@apache.org.
HDFS-9764. DistCp doesn't print value for several arguments including -numListstatusThreads. (Wei-Chiu Chuang via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba1c9d48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba1c9d48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba1c9d48
Branch: refs/heads/HDFS-1312
Commit: ba1c9d484a8d6a18b4b7365c49d09635d24e579a
Parents: a963bab
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Fri Feb 19 10:17:37 2016 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Fri Feb 19 10:17:37 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/tools/DistCpOptions.java | 10 +++++++++-
.../org/apache/hadoop/tools/TestOptionsParser.java | 15 ++++++++++-----
3 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba1c9d48/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2129a69..1f5d65d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1942,6 +1942,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8923. Add -source flag to balancer usage message. (Chris Trezzo
via szetszwo)
+ HDFS-9764. DistCp doesn't print value for several arguments including
+ -numListstatusThreads. (Wei-Chiu Chuang via Yongjun Zhang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba1c9d48/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 10cf388..cbd4ac0 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -664,14 +664,22 @@ public class DistCpOptions {
", syncFolder=" + syncFolder +
", deleteMissing=" + deleteMissing +
", ignoreFailures=" + ignoreFailures +
+ ", overwrite=" + overwrite +
+ ", skipCRC=" + skipCRC +
+ ", blocking=" + blocking +
+ ", numListstatusThreads=" + numListstatusThreads +
", maxMaps=" + maxMaps +
+ ", mapBandwidth=" + mapBandwidth +
", sslConfigurationFile='" + sslConfigurationFile + '\'' +
", copyStrategy='" + copyStrategy + '\'' +
+ ", preserveStatus=" + preserveStatus +
+ ", preserveRawXattrs=" + preserveRawXattrs +
+ ", atomicWorkPath=" + atomicWorkPath +
+ ", logPath=" + logPath +
", sourceFileListing=" + sourceFileListing +
", sourcePaths=" + sourcePaths +
", targetPath=" + targetPath +
", targetPathExists=" + targetPathExists +
- ", preserveRawXattrs=" + preserveRawXattrs +
", filtersFile='" + filtersFile + '\'' +
'}';
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba1c9d48/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index 78ae415..4c136a8 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -399,11 +399,16 @@ public class TestOptionsParser {
@Test
public void testToString() {
DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz"));
- String val = "DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false, " +
- "ignoreFailures=false, maxMaps=20, sslConfigurationFile='null', copyStrategy='uniformsize', " +
- "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, targetPathExists=true, " +
- "preserveRawXattrs=false, filtersFile='null'}";
- Assert.assertEquals(val, option.toString());
+ String val = "DistCpOptions{atomicCommit=false, syncFolder=false, "
+ + "deleteMissing=false, ignoreFailures=false, overwrite=false, "
+ + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
+ + "mapBandwidth=100.0, sslConfigurationFile='null', "
+ + "copyStrategy='uniformsize', preserveStatus=[], "
+ + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
+ + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
+ + "targetPathExists=true, filtersFile='null'}";
+ String optionString = option.toString();
+ Assert.assertEquals(val, optionString);
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
DistCpOptionSwitch.ATOMIC_COMMIT.name());
}
[27/50] [abbrv] hadoop git commit: HDFS-4946. Allow preferLocalNode
in BlockPlacementPolicyDefault to be configurable. Contributed by James
Kinley and Nathan Roberts.
Posted by ar...@apache.org.
HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be configurable. Contributed by James Kinley and Nathan Roberts.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba6d5ed9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba6d5ed9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba6d5ed9
Branch: refs/heads/HDFS-1312
Commit: ba6d5ed955aa34cdcb7aaa4b1025da6ee5e91355
Parents: a365a39
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Feb 18 13:21:51 2016 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Feb 18 13:21:51 2016 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++
.../blockmanagement/BlockPlacementPolicyDefault.java | 7 ++++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 12 ++++++++++++
4 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba6d5ed9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2948a94..b6510b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2811,6 +2811,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9669. TcpPeerServer should respect ipc.server.listen.queue.size
(Elliot Clark via cmccabe)
+ HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be
+ configurable (James Kinley and Nathan Roberts via kihwal)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba6d5ed9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index df205db..39e4fe4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -663,6 +663,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction";
public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT =
0.6f;
+ public static final String DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY =
+ "dfs.namenode.block-placement-policy.default.prefer-local-node";
+ public static final boolean DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT = true;
public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user";
public static final String DFS_DOMAIN_SOCKET_PATH_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba6d5ed9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 14439e7..ee891a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -59,7 +59,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
protected boolean considerLoad;
protected double considerLoadFactor;
- private boolean preferLocalNode = true;
+ private boolean preferLocalNode;
protected NetworkTopology clusterMap;
protected Host2NodesMap host2datanodeMap;
private FSClusterStats stats;
@@ -95,6 +95,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
this.staleInterval = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT);
+ this.preferLocalNode = conf.getBoolean(
+ DFSConfigKeys.
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY,
+ DFSConfigKeys.
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba6d5ed9/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0f5130a..1560335 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -922,6 +922,18 @@
</property>
<property>
+ <name>dfs.namenode.block-placement-policy.default.prefer-local-node</name>
+ <value>true</value>
+ <description>Controls how the default block placement policy places
+ the first replica of a block. When true, it will prefer the node where
+ the client is running. When false, it will prefer a node in the same rack
+ as the client. Setting to false avoids situations where entire copies of
+ large files end up on a single node, thus creating hotspots.
+ </description>
+</property>
+
+
+<property>
<name>dfs.stream-buffer-size</name>
<value>4096</value>
<description>The size of buffer to stream files.
[18/50] [abbrv] hadoop git commit: YARN-4689. FairScheduler: Cleanup
preemptContainer to be more readable. (Kai Sasaki via kasha)
Posted by ar...@apache.org.
YARN-4689. FairScheduler: Cleanup preemptContainer to be more readable. (Kai Sasaki via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ab4c476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ab4c476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ab4c476
Branch: refs/heads/HDFS-1312
Commit: 2ab4c476ed22d3ccf15b215710b67e32bbc7e5f0
Parents: 0fb14aa
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Feb 17 17:44:35 2016 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Feb 17 18:16:15 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/scheduler/fair/FSParentQueue.java | 6 ++----
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab4c476/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1547f10..f7c69a4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -34,6 +34,9 @@ Trunk - Unreleased
YARN-3261. rewrite resourcemanager restart doc to remove roadmap bits (Gururaj Shetty via aw)
+ YARN-4689. FairScheduler: Cleanup preemptContainer to be more readable.
+ (Kai Sasaki via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ab4c476/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index a028422..d9fac90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -266,10 +266,8 @@ public class FSParentQueue extends FSQueue {
LOG.debug("skipping from queue=" + getName()
+ " because it's a non-preemptable queue");
}
- continue;
- }
- if (candidateQueue == null ||
- comparator.compare(queue, candidateQueue) > 0) {
+ } else if (candidateQueue == null ||
+ comparator.compare(queue, candidateQueue) > 0) {
candidateQueue = queue;
}
}
[13/50] [abbrv] hadoop git commit: HADOOP-12878. KMS SPNEGO sequence
does not work with WEBHDFS. Contributed by Xiaoyu Yao.
Posted by ar...@apache.org.
HADOOP-12878. KMS SPNEGO sequence does not work with WEBHDFS. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd1befb6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd1befb6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd1befb6
Branch: refs/heads/HDFS-1312
Commit: fd1befb6ba450e45b1fcb3fb28b0da6c48daf6b3
Parents: 1a1c2f5
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Feb 17 08:27:27 2016 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Feb 17 08:27:27 2016 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop/crypto/key/kms/KMSClientProvider.java | 14 +++++++++-----
2 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd1befb6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b0dda76..cde5f3b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1710,6 +1710,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12780. During atomic rename handle crash when one directory has been
renamed but not file under it. (Madhumita Chakraborty via cnauroth)
+ HADOOP-12878. KMS SPNEGO sequence does not work with WEBHDFS. (xyao)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd1befb6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index fd7a83e..7682888 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -397,11 +397,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
new EncryptedQueueRefiller());
authToken = new DelegationTokenAuthenticatedURL.Token();
- actualUgi =
- (UserGroupInformation.getCurrentUser().getAuthenticationMethod() ==
- UserGroupInformation.AuthenticationMethod.PROXY) ? UserGroupInformation
- .getCurrentUser().getRealUser() : UserGroupInformation
- .getCurrentUser();
+ UserGroupInformation.AuthenticationMethod authMethod =
+ UserGroupInformation.getCurrentUser().getAuthenticationMethod();
+ if (authMethod == UserGroupInformation.AuthenticationMethod.PROXY) {
+ actualUgi = UserGroupInformation.getCurrentUser().getRealUser();
+ } else if (authMethod == UserGroupInformation.AuthenticationMethod.TOKEN) {
+ actualUgi = UserGroupInformation.getLoginUser();
+ } else {
+ actualUgi =UserGroupInformation.getCurrentUser();
+ }
}
private static Path extractKMSPath(URI uri) throws MalformedURLException, IOException {
[08/50] [abbrv] hadoop git commit: HDFS-9456.
BlockPlacementPolicyWithNodeGroup should override verifyBlockPlacement().
Contributed by Xiaobing Zhou.
Posted by ar...@apache.org.
HDFS-9456. BlockPlacementPolicyWithNodeGroup should override verifyBlockPlacement(). Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77ba5add
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77ba5add
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77ba5add
Branch: refs/heads/HDFS-1312
Commit: 77ba5add0d9cb10d45ca9122bca48baa7c8fb3b8
Parents: 4b0e59f
Author: Junping Du <ju...@apache.org>
Authored: Tue Feb 16 18:55:55 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Tue Feb 16 18:55:55 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../BlockPlacementPolicyWithNodeGroup.java | 46 +++++++++
.../BlockPlacementStatusWithNodeGroup.java | 81 +++++++++++++++
.../TestReplicationPolicyWithNodeGroup.java | 100 +++++++++++++++++++
4 files changed, 230 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77ba5add/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e9160c9..0b220bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2768,6 +2768,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9801. ReconfigurableBase should update the cached configuration.
(Arpit Agarwal)
+ HDFS-9456. BlockPlacementPolicyWithNodeGroup should override
+ verifyBlockPlacement(). (Xiaobing Zhou via junping_du)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77ba5add/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
index c62d977..194f6ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
@@ -389,4 +389,50 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
}
return true;
}
+
+
+ @Override
+ public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
+ int numberOfReplicas) {
+ if (locs == null) {
+ locs = DatanodeDescriptor.EMPTY_ARRAY;
+ }
+
+ List<String> locList = new ArrayList<String>();
+ /*
+ * remove the part of node group for BlockPlacementPolicyDefault to count
+ * distinct racks, e.g. "/d1/r1/n1" --> "/d1/r1"
+ */
+ for (int i = 0; i < locs.length; i++) {
+ locList.add(locs[i].getNetworkLocation());
+ locs[i].setNetworkLocation(NetworkTopology.getFirstHalf(locs[i]
+ .getNetworkLocation()));
+ }
+
+ BlockPlacementStatus defaultStatus = super.verifyBlockPlacement(locs,
+ numberOfReplicas);
+
+ // restore the part of node group back
+ for (int i = 0; i < locs.length; i++) {
+ locs[i].setNetworkLocation(locList.get(i));
+ }
+
+ int minNodeGroups = numberOfReplicas;
+ BlockPlacementStatusWithNodeGroup nodeGroupStatus =
+ new BlockPlacementStatusWithNodeGroup(
+ defaultStatus, getNodeGroupsFromNode(locs), minNodeGroups);
+ return nodeGroupStatus;
+ }
+
+ private Set<String> getNodeGroupsFromNode(DatanodeInfo[] nodes) {
+ Set<String> nodeGroups = new HashSet<>();
+ if (nodes == null) {
+ return nodeGroups;
+ }
+
+ for (DatanodeInfo node : nodes) {
+ nodeGroups.add(NetworkTopology.getLastHalf(node.getNetworkLocation()));
+ }
+ return nodeGroups;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77ba5add/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusWithNodeGroup.java
new file mode 100644
index 0000000..b98b3da
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusWithNodeGroup.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An implementation of @see BlockPlacementStatus for
+ * @see BlockPlacementPolicyWithNodeGroup
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BlockPlacementStatusWithNodeGroup implements BlockPlacementStatus {
+
+ private final BlockPlacementStatus parentBlockPlacementStatus;
+ private final Set<String> currentNodeGroups;
+ private final int requiredNodeGroups;
+
+ /**
+ * @param parentBlockPlacementStatus the parent class' status
+ * @param currentNodeGroups the current set of node groups of the replicas
+ * @param requiredNodeGroups the number of required node groups
+ */
+ public BlockPlacementStatusWithNodeGroup(
+ BlockPlacementStatus parentBlockPlacementStatus,
+ Set<String> currentNodeGroups, int requiredNodeGroups) {
+ this.parentBlockPlacementStatus = parentBlockPlacementStatus;
+ this.currentNodeGroups = currentNodeGroups;
+ this.requiredNodeGroups = requiredNodeGroups;
+ }
+
+ @Override
+ public boolean isPlacementPolicySatisfied() {
+ return parentBlockPlacementStatus.isPlacementPolicySatisfied()
+ && isNodeGroupPolicySatisfied();
+ }
+
+ private boolean isNodeGroupPolicySatisfied() {
+ return requiredNodeGroups <= currentNodeGroups.size();
+ }
+
+ @Override
+ public String getErrorDescription() {
+ if (isPlacementPolicySatisfied()) {
+ return null;
+ }
+
+ StringBuilder errorDescription = new StringBuilder();
+ if (!parentBlockPlacementStatus.isPlacementPolicySatisfied()) {
+ errorDescription.append(parentBlockPlacementStatus.getErrorDescription());
+ }
+
+ if (!isNodeGroupPolicySatisfied()) {
+ if (errorDescription.length() != 0) {
+ errorDescription.append(" ");
+ }
+ errorDescription.append("The block has " + requiredNodeGroups
+ + " replicas. But it only has " + currentNodeGroups.size()
+ + " node groups " + currentNodeGroups + ".");
+ }
+ return errorDescription.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77ba5add/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index 7a00a3b..edcab10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -34,6 +34,9 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.NetworkTopologyWithNodeGroup;
@@ -130,6 +133,103 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
};
/**
+ * Test block placement verification.
+ * @throws Exception
+ */
+ @Test
+ public void testVerifyBlockPlacement() throws Exception {
+ LocatedBlock locatedBlock;
+ BlockPlacementStatus status;
+ ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
+ List<DatanodeStorageInfo> set = new ArrayList<>();
+
+ // 2 node groups (not enough), 2 racks (enough)
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[1]);
+ set.add(storages[4]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertFalse(status.isPlacementPolicySatisfied());
+
+ // 3 node groups (enough), 2 racks (enough)
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[2]);
+ set.add(storages[5]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertTrue(status.isPlacementPolicySatisfied());
+
+ // 2 node groups (not enough), 1 rack (not enough)
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[1]);
+ set.add(storages[2]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertFalse(status.isPlacementPolicySatisfied());
+ assertTrue(status.getErrorDescription().contains("node group"));
+ assertTrue(status.getErrorDescription().contains("more rack(s)"));
+
+ // 3 node groups (enough), 3 racks (enough)
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[5]);
+ set.add(storages[7]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertTrue(status.isPlacementPolicySatisfied());
+
+ // 3 node groups (not enough), 3 racks (enough), 4 replicas
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[1]);
+ set.add(storages[5]);
+ set.add(storages[7]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertFalse(status.isPlacementPolicySatisfied());
+ assertTrue(status.getErrorDescription().contains("node group"));
+ assertFalse(status.getErrorDescription().contains("more rack(s)"));
+
+ // 2 node groups (not enough), 1 rack (not enough)
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[1]);
+ set.add(storages[2]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertFalse(status.isPlacementPolicySatisfied());
+ assertTrue(status.getErrorDescription().contains("node group"));
+ assertTrue(status.getErrorDescription().contains("more rack(s)"));
+
+ // 1 node group (not enough), 1 rack (not enough)
+ set.clear();
+ set.add(storages[0]);
+ set.add(storages[1]);
+ locatedBlock = BlockManager.newLocatedBlock(b,
+ set.toArray(new DatanodeStorageInfo[set.size()]), 0, false);
+ status = replicator.verifyBlockPlacement(locatedBlock.getLocations(),
+ set.size());
+ assertFalse(status.isPlacementPolicySatisfied());
+ assertTrue(status.getErrorDescription().contains("node group"));
+ assertTrue(status.getErrorDescription().contains("more rack(s)"));
+ }
+
+ /**
* Scan the targets list: all targets should be on different NodeGroups.
* Return false if two targets are found on the same NodeGroup.
*/
[10/50] [abbrv] hadoop git commit: HDFS-9787. SNNs stop uploading
FSImage to ANN once isPrimaryCheckPointer changed to false. (Contributed by
Guocui Mi)
Posted by ar...@apache.org.
HDFS-9787. SNNs stop uploading FSImage to ANN once isPrimaryCheckPointer changed to false. (Contributed by Guocui Mi)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2536ece7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2536ece7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2536ece7
Branch: refs/heads/HDFS-1312
Commit: 2536ece7b67ccfeac44314615a5f9dbe771ce373
Parents: 96ea309
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Feb 17 10:41:24 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Feb 17 10:41:24 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/namenode/ha/StandbyCheckpointer.java | 7 +++-
.../namenode/ha/TestStandbyCheckpoints.java | 42 ++++++++++++++++++++
3 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2536ece7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0b220bf..79369da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -935,6 +935,9 @@ Trunk (Unreleased)
HDFS-9755. Erasure Coding: allow to use multiple EC policies in striping
related tests [Part 2]. (Rui Li via zhz)
+ HDFS-9787. SNNs stop uploading FSImage to ANN once isPrimaryCheckPointer
+ changed to false. (Guocui Mi via vinayakumarb)
+
Release 2.9.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2536ece7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
index f5ecbec..099a6aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
@@ -65,6 +65,7 @@ public class StandbyCheckpointer {
private final Configuration conf;
private final FSNamesystem namesystem;
private long lastCheckpointTime;
+ private long lastUploadTime;
private final CheckpointerThread thread;
private final ThreadFactory uploadThreadFactory;
private List<URL> activeNNAddresses;
@@ -252,6 +253,7 @@ public class StandbyCheckpointer {
break;
}
}
+ lastUploadTime = monotonicNow();
// we are primary if we successfully updated the ANN
this.isPrimaryCheckPointer = success;
@@ -362,6 +364,7 @@ public class StandbyCheckpointer {
// Reset checkpoint time so that we don't always checkpoint
// on startup.
lastCheckpointTime = monotonicNow();
+ lastUploadTime = monotonicNow();
while (shouldRun) {
boolean needRollbackCheckpoint = namesystem.isNeedRollbackFsImage();
if (!needRollbackCheckpoint) {
@@ -414,7 +417,9 @@ public class StandbyCheckpointer {
// on all nodes, we build the checkpoint. However, we only ship the checkpoint if have a
// rollback request, are the checkpointer, are outside the quiet period.
- boolean sendRequest = isPrimaryCheckPointer || secsSinceLast >= checkpointConf.getQuietPeriod();
+ final long secsSinceLastUpload = (now - lastUploadTime) / 1000;
+ boolean sendRequest = isPrimaryCheckPointer
+ || secsSinceLastUpload >= checkpointConf.getQuietPeriod();
doCheckpoint(sendRequest);
// reset needRollbackCheckpoint to false only when we finish a ckpt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2536ece7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
index 37d346a..bdeeab5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
@@ -458,6 +458,48 @@ public class TestStandbyCheckpoints {
t.join();
}
+ /**
+ * Test for the case standby NNs can upload FSImage to ANN after
+ * become non-primary standby NN. HDFS-9787
+ */
+ @Test(timeout=300000)
+ public void testNonPrimarySBNUploadFSImage() throws Exception {
+ // Shutdown all standby NNs.
+ for (int i = 1; i < NUM_NNS; i++) {
+ cluster.shutdownNameNode(i);
+
+ // Checkpoint as fast as we can, in a tight loop.
+ cluster.getConfiguration(i).setInt(
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
+ }
+
+ doEdits(0, 10);
+ cluster.transitionToStandby(0);
+
+ // Standby NNs do checkpoint without active NN available.
+ for (int i = 1; i < NUM_NNS; i++) {
+ cluster.restartNameNode(i, false);
+ }
+ cluster.waitClusterUp();
+
+ for (int i = 0; i < NUM_NNS; i++) {
+ // Once the standby catches up, it should do a checkpoint
+ // and save to local directories.
+ HATestUtil.waitForCheckpoint(cluster, 1, ImmutableList.of(12));
+ }
+
+ cluster.transitionToActive(0);
+
+ // Wait for 2 seconds to expire last upload time.
+ Thread.sleep(2000);
+
+ doEdits(11, 20);
+ nns[0].getRpcServer().rollEditLog();
+
+ // One of standby NNs should also upload it back to the active.
+ HATestUtil.waitForCheckpoint(cluster, 0, ImmutableList.of(23));
+ }
+
private void doEdits(int start, int stop) throws IOException {
for (int i = start; i < stop; i++) {
Path p = new Path("/test" + i);
[03/50] [abbrv] hadoop git commit: HADOOP-12786. "hadoop key" command
usage is not documented. Contributed by Xiao Chen.
Posted by ar...@apache.org.
HADOOP-12786. "hadoop key" command usage is not documented. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c48e50c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c48e50c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c48e50c
Branch: refs/heads/HDFS-1312
Commit: 1c48e50ce71fd0e38740d69ce1e03119dca97a38
Parents: dea90c9
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 15 11:38:25 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 15 11:38:25 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/key/KeyShell.java | 2 +-
.../hadoop-common/src/site/markdown/CommandsManual.md | 14 +++++++++++++-
3 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c48e50c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a05fd4b..0d07e36 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1809,6 +1809,9 @@ Release 2.7.2 - 2016-01-25
HADOOP-12415. Fixed pom files to correctly include compile-time dependency on
netty. (Tom Zeng via cos)
+ HADOOP-12786. "hadoop key" command usage is not documented.
+ (Xiao Chen via aajisaka)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c48e50c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index 4c72d06..c69dc82 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -372,7 +372,7 @@ public class KeyShell extends Configured implements Tool {
public static final String DESC =
"The delete subcommand deletes all versions of the key\n" +
"specified by the <keyname> argument from within the\n" +
- "provider specified -provider. The command asks for\n" +
+ "provider specified by -provider. The command asks for\n" +
"user confirmation unless -f is specified.";
String keyName = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c48e50c/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index d7f0657..58fcbe9 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -188,7 +188,19 @@ Example: `hadoop kerbname user@EXAMPLE.COM`
### `key`
-Manage keys via the KeyProvider.
+Usage: `hadoop key <subcommand> [options]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| create *keyname* [-cipher *cipher*] [-size *size*] [-description *description*] [-attr *attribute=value*] [-provider *provider*] [-help] | Creates a new key for the name specified by the *keyname* argument within the provider specified by the `-provider` argument. You may specify a cipher with the `-cipher` argument. The default cipher is currently "AES/CTR/NoPadding". The default keysize is 128. You may specify the requested key length using the `-size` argument. Arbitrary attribute=value style attributes may be specified using the `-attr` argument. `-attr` may be specified multiple times, once per attribute. |
+| roll *keyname* [-provider *provider*] [-help] | Creates a new version for the specified key within the provider indicated using the `-provider` argument |
+| delete *keyname* [-provider *provider*] [-f] [-help] | Deletes all versions of the key specified by the *keyname* argument from within the provider specified by `-provider`. The command asks for user confirmation unless `-f` is specified. |
+| list [-provider *provider*] [-metadata] [-help] | Displays the keynames contained within a particular provider as configured in core-site.xml or specified with the `-provider` argument. `-metadata` displays the metadata. |
+| -help | Prints usage of this command |
+
+Manage keys via the KeyProvider. For details on KeyProviders, see the [Transparent Encryption Guide](../hadoop-hdfs/TransparentEncryption.html).
+
+NOTE: Some KeyProviders (e.g. org.apache.hadoop.crypto.key.JavaKeyStoreProvider) does not support uppercase key names.
### `trace`
[33/50] [abbrv] hadoop git commit: HDFS-8923. Add -source flag to
balancer usage message. Contributed by Chris Trezzo
Posted by ar...@apache.org.
HDFS-8923. Add -source flag to balancer usage message. Contributed by Chris Trezzo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46f77902
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46f77902
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46f77902
Branch: refs/heads/HDFS-1312
Commit: 46f77902368e5d8cc2d3e24de735b98a83ad46e1
Parents: 8eee59c
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Feb 18 18:18:12 2016 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Feb 18 18:18:12 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/balancer/Balancer.java | 2 ++
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 ++
3 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f77902/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d54aa96..945c911 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2817,6 +2817,9 @@ Release 2.7.3 - UNRELEASED
HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be
configurable (James Kinley and Nathan Roberts via kihwal)
+ HDFS-8923. Add -source flag to balancer usage message. (Chris Trezzo
+ via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f77902/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 191e1c3..49980ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -179,6 +179,8 @@ public class Balancer {
+ "\tExcludes the specified datanodes."
+ "\n\t[-include [-f <hosts-file> | <comma-separated list of hosts>]]"
+ "\tIncludes only the specified datanodes."
+ + "\n\t[-source [-f <hosts-file> | <comma-separated list of hosts>]]"
+ + "\tPick only the specified datanodes as source nodes."
+ "\n\t[-blockpools <comma-separated list of blockpool ids>]"
+ "\tThe balancer will only run on blockpools included in this list."
+ "\n\t[-idleiterations <idleiterations>]"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f77902/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 6584669..dc6e935 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -267,6 +267,7 @@ Usage:
[-policy <policy>]
[-exclude [-f <hosts-file> | <comma-separated list of hosts>]]
[-include [-f <hosts-file> | <comma-separated list of hosts>]]
+ [-source [-f <hosts-file> | <comma-separated list of hosts>]]
[-blockpools <comma-separated list of blockpool ids>]
[-idleiterations <idleiterations>]
@@ -276,6 +277,7 @@ Usage:
| `-threshold` \<threshold\> | Percentage of disk capacity. This overwrites the default threshold. |
| `-exclude -f` \<hosts-file\> \| \<comma-separated list of hosts\> | Excludes the specified datanodes from being balanced by the balancer. |
| `-include -f` \<hosts-file\> \| \<comma-separated list of hosts\> | Includes only the specified datanodes to be balanced by the balancer. |
+| `-source -f` \<hosts-file\> \| \<comma-separated list of hosts\> | Pick only the specified datanodes as source nodes. |
| `-blockpools` \<comma-separated list of blockpool ids\> | The balancer will only run on blockpools included in this list. |
| `-idleiterations` \<iterations\> | Maximum number of idle iterations before exit. This overwrites the default idleiterations(5). |
[44/50] [abbrv] hadoop git commit: HDFS-7452. skip StandbyException
log for getCorruptFiles() (Contributed by Bramha Reddy Battula)
Posted by ar...@apache.org.
HDFS-7452. skip StandbyException log for getCorruptFiles() (Contributed by Bramha Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3135167
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3135167
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3135167
Branch: refs/heads/HDFS-1312
Commit: f313516731d787f6be64c9406ca83d941d47ee99
Parents: b68901d
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Feb 22 09:21:46 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Feb 22 09:21:46 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3135167/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9215488..9a156a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2804,6 +2804,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9839. Reduce verbosity of processReport logging. (Arpit Agarwal)
+ HDFS-7452. skip StandbyException log for getCorruptFiles()
+ (Bramha Reddy Battula via vinayakumarb)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3135167/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4722ac1..64f23dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5596,6 +5596,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
list.add(c.toString());
}
}
+ } catch (StandbyException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Get corrupt file blocks returned error: " + e.getMessage());
+ }
} catch (IOException e) {
LOG.warn("Get corrupt file blocks returned error: " + e.getMessage());
}
[22/50] [abbrv] hadoop git commit: YARN-4654. Yarn node label CLI
should parse "=" correctly when trying to remove all labels on a node.
(Naganarasimha G R via rohithsharmaks)
Posted by ar...@apache.org.
YARN-4654. Yarn node label CLI should parse "=" correctly when trying to remove all labels on a node. (Naganarasimha G R via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1afac3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1afac3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1afac3a
Branch: refs/heads/HDFS-1312
Commit: c1afac3a98a4889d30a6ec7574a364fcbb6bb5b0
Parents: 7de7068
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Feb 18 14:10:37 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Thu Feb 18 14:10:37 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/client/cli/RMAdminCLI.java | 25 ++++++++-------
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 32 +++++++++++++++++++-
3 files changed, 48 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1afac3a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fa0ba44..9b64204 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1428,6 +1428,9 @@ Release 2.8.0 - UNRELEASED
YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
is configured. (Naganarasimha G R via devaraj)
+ YARN-4654. Yarn node label CLI should parse "=" correctly when trying to
+ remove all labels on a node. (Naganarasimha G R via rohithsharmaks)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1afac3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 5c3b1d8..d407c20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -59,8 +59,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioning
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
@@ -592,26 +592,29 @@ public class RMAdminCLI extends HAAdmin {
continue;
}
- // "," also supported for compatibility
String[] splits = nodeToLabels.split("=");
- int index = 0;
- if (splits.length != 2) {
- splits = nodeToLabels.split(",");
- index = 1;
- }
-
+ int labelsStartIndex = 0;
String nodeIdStr = splits[0];
- if (index == 0) {
+
+ if (splits.length == 2) {
splits = splits[1].split(",");
+ } else if (nodeToLabels.endsWith("=")) {
+ //case where no labels are mapped to a node
+ splits = new String[0];
+ } else {
+ // "," also supported for compatibility
+ splits = nodeToLabels.split(",");
+ nodeIdStr = splits[0];
+ labelsStartIndex = 1;
}
-
+
Preconditions.checkArgument(!nodeIdStr.trim().isEmpty(),
"node name cannot be empty");
NodeId nodeId = ConverterUtils.toNodeIdWithDefaultPort(nodeIdStr);
map.put(nodeId, new HashSet<String>());
- for (int i = index; i < splits.length; i++) {
+ for (int i = labelsStartIndex; i < splits.length; i++) {
if (!splits[i].trim().isEmpty()) {
map.get(nodeId).add(splits[i].trim());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1afac3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index 08e63ea..057594d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -36,7 +36,9 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
@@ -145,7 +147,20 @@ public class TestRMAdminCLI {
private void initDummyNodeLabelsManager() {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
- dummyNodeLabelsManager = new DummyCommonNodeLabelsManager();
+ dummyNodeLabelsManager = new DummyCommonNodeLabelsManager() {
+ @Override
+ public void replaceLabelsOnNode(
+ Map<NodeId, Set<String>> replaceLabelsToNode) throws IOException {
+ Iterator<NodeId> iterator = replaceLabelsToNode.keySet().iterator();
+ while(iterator.hasNext()) {
+ NodeId nodeId=iterator.next();
+ if(nodeId.getHost().endsWith("=")){
+ throw new IOException("Parsing of Input String failed");
+ }
+ }
+ super.replaceLabelsOnNode(replaceLabelsToNode);
+ }
+ };
dummyNodeLabelsManager.init(conf);
}
@@ -712,6 +727,21 @@ public class TestRMAdminCLI {
assertTrue(0 != rmAdminCLI.run(args));
}
+ @Test
+ public void testRemoveLabelsOnNodes() throws Exception {
+ // Successfully replace labels
+ dummyNodeLabelsManager
+ .addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
+ String[] args = { "-replaceLabelsOnNode", "node1=x node2=y",
+ "-directlyAccessNodeLabelStore" };
+ assertTrue(0 == rmAdminCLI.run(args));
+
+ args = new String[] { "-replaceLabelsOnNode", "node1= node2=",
+ "-directlyAccessNodeLabelStore" };
+ assertTrue("Labels should get replaced even '=' is used ",
+ 0 == rmAdminCLI.run(args));
+ }
+
private void testError(String[] args, String template,
ByteArrayOutputStream data, int resultCode) throws Exception {
int actualResultCode = rmAdminCLI.run(args);
[12/50] [abbrv] hadoop git commit: HADOOP-12710. Remove dependency on
commons-httpclient for TestHttpServerLogs (Wei-Chiu Chuang via iwasakims)
Posted by ar...@apache.org.
HADOOP-12710. Remove dependency on commons-httpclient for TestHttpServerLogs (Wei-Chiu Chuang via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a1c2f5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a1c2f5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a1c2f5b
Branch: refs/heads/HDFS-1312
Commit: 1a1c2f5bab522ed39a35edbca30e6bed5a990992
Parents: 453e7e0
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Thu Feb 18 00:13:11 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Thu Feb 18 00:13:11 2016 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/test/java/org/apache/hadoop/http/TestHttpServerLogs.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1c2f5b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0aa39d9..b0dda76 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -688,6 +688,9 @@ Release 2.9.0 - UNRELEASED
HADOOP-12749. Create a threadpoolexecutor that overrides afterExecute to
log uncaught exceptions/errors. (Sidharta Seethana via vvasudev)
+ HADOOP-12710. Remove dependency on commons-httpclient for TestHttpServerLogs
+ (Wei-Chiu Chuang via iwasakims)
+
BUG FIXES
HADOOP-12605. Fix intermittent failure of TestIPC.testIpcWithReaderQueuing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1c2f5b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLogs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLogs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLogs.java
index 46ea095..d72a958 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLogs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLogs.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.http;
-import org.apache.commons.httpclient.HttpStatus;
+import org.apache.http.HttpStatus;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
[06/50] [abbrv] hadoop git commit: HDFS-9765.
TestBlockScanner#testVolumeIteratorWithCaching fails intermittently.
(aajisaka)
Posted by ar...@apache.org.
HDFS-9765. TestBlockScanner#testVolumeIteratorWithCaching fails intermittently. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ed07bd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ed07bd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ed07bd3
Branch: refs/heads/HDFS-1312
Commit: 8ed07bd3e6b6a8af3b82e9a085c74be29f8dfd87
Parents: b0738ae
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 15 19:08:03 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 15 19:08:03 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed07bd3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d144ba7..e9160c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2850,6 +2850,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9784. Example usage is not correct in Transparent Encryption document.
(Takashi Ohnishi via aajisaka)
+ HDFS-9765. TestBlockScanner#testVolumeIteratorWithCaching fails
+ intermittently. (aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed07bd3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
index acd5a47..4628a46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
@@ -245,7 +245,7 @@ public class TestBlockScanner {
testVolumeIteratorImpl(5, 0);
}
- @Test(timeout=60000)
+ @Test(timeout=300000)
public void testVolumeIteratorWithCaching() throws Exception {
testVolumeIteratorImpl(600, 100);
}
[07/50] [abbrv] hadoop git commit: HADOOP-12805. Annotate CanUnbuffer
with @InterfaceAudience.Public (Contributed by Ted Yu)
Posted by ar...@apache.org.
HADOOP-12805. Annotate CanUnbuffer with @InterfaceAudience.Public (Contributed by Ted Yu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b0e59fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b0e59fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b0e59fd
Branch: refs/heads/HDFS-1312
Commit: 4b0e59fdc010be517dd67ddf35c962da204df547
Parents: 8ed07bd
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Feb 16 17:32:03 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Feb 16 17:32:03 2016 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/CanUnbuffer.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0e59fd/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0d07e36..1a2b32f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -2550,6 +2550,9 @@ Release 2.6.5 - UNRELEASED
IMPROVEMENTS
+ HADOOP-12805. Annotate CanUnbuffer with @InterfaceAudience.Public
+ (Ted Yu via vinayakumarb)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b0e59fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CanUnbuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CanUnbuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CanUnbuffer.java
index 07e65f5..e322798 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CanUnbuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CanUnbuffer.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* FSDataInputStreams implement this interface to indicate that they can clear
* their buffers on request.
*/
-@InterfaceAudience.Private
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface CanUnbuffer {
/**
[24/50] [abbrv] hadoop git commit: HDFS-6832. Fix the usage of 'hdfs
namenode' command. Contributed by Manjunath Ballur.
Posted by ar...@apache.org.
HDFS-6832. Fix the usage of 'hdfs namenode' command. Contributed by Manjunath Ballur.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e78d8e66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e78d8e66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e78d8e66
Branch: refs/heads/HDFS-1312
Commit: e78d8e66f748c68d2d88ee748facbfa35817d5e3
Parents: 2440671
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Feb 19 01:35:06 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Feb 19 01:35:06 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/namenode/NameNode.java | 3 +--
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e78d8e66/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 89fc187..61a783d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2782,6 +2782,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9799. Reimplement getCurrentTrashDir to remove incompatibility. (zhz)
+ HDFS-6832. Fix the usage of 'hdfs namenode' command.
+ (Manjunath Ballur via aajisaka)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e78d8e66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 7c4e10f..f376901 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -282,8 +282,7 @@ public class NameNode implements NameNodeStatusMXBean {
+ StartupOption.BOOTSTRAPSTANDBY.getName() + "] | \n\t["
+ StartupOption.RECOVER.getName() + " [ "
+ StartupOption.FORCE.getName() + "] ] | \n\t["
- + StartupOption.METADATAVERSION.getName() + " ] "
- + " ]";
+ + StartupOption.METADATAVERSION.getName() + " ]";
public long getProtocolVersion(String protocol,
[29/50] [abbrv] hadoop git commit: HDFS-9830. Remove references to
hftp in ViewFs documentation in trunk. Contributed by Wei-Chiu Chuang.
Posted by ar...@apache.org.
HDFS-9830. Remove references to hftp in ViewFs documentation in trunk. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7ac25e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7ac25e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7ac25e5
Branch: refs/heads/HDFS-1312
Commit: c7ac25e52c307756c30157377fb52eced3aed5e4
Parents: bbfaf3c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Feb 19 05:02:09 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Feb 19 05:02:57 2016 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/FileSystemShell.md | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md | 8 ++++----
3 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ac25e5/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index ab66156..a356b8a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -60,7 +60,7 @@
Overview
========
-The File System (FS) shell includes various shell-like commands that directly interact with the Hadoop Distributed File System (HDFS) as well as other file systems that Hadoop supports, such as Local FS, HFTP FS, S3 FS, and others. The FS shell is invoked by:
+The File System (FS) shell includes various shell-like commands that directly interact with the Hadoop Distributed File System (HDFS) as well as other file systems that Hadoop supports, such as Local FS, WebHDFS, S3 FS, and others. The FS shell is invoked by:
bin/hadoop fs <args>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ac25e5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b6510b0..d54aa96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -938,6 +938,9 @@ Trunk (Unreleased)
HDFS-9787. SNNs stop uploading FSImage to ANN once isPrimaryCheckPointer
changed to false. (Guocui Mi via vinayakumarb)
+ HDFS-9830. Remove references to hftp in ViewFs documentation in trunk.
+ (Wei-Chiu Chuang via aajisaka)
+
Release 2.9.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ac25e5/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
index ef5976c..067265a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
@@ -75,9 +75,9 @@ Hence on Cluster X where the `core-site.xml` is set as above, the typical pathna
distcp hdfs://namenodeClusterY:port/pathSrc hdfs://namenodeClusterZ:port/pathDest
-4. `webhdfs://namenodeClusterX:http_port/foo/bar` and `hftp://namenodeClusterX:http_port/foo/bar`
+4. `webhdfs://namenodeClusterX:http_port/foo/bar`
- * These are file system URIs respectively for accessing files via the WebHDFS file system and the HFTP file system. Note that WebHDFS and HFTP use the HTTP port of the namenode but not the RPC port.
+ * It is an URI for accessing files via the WebHDFS file system. Note that WebHDFS uses the HTTP port of the namenode but not the RPC port.
5. `http://namenodeClusterX:http_port/webhdfs/v1/foo/bar` and `http://proxyClusterX:http_port/foo/bar`
@@ -133,9 +133,9 @@ Hence on Cluster X, where the `core-site.xml` is set to make the default fs to u
distcp viewfs://clusterY:/pathSrc viewfs://clusterZ/pathDest
-4. `viewfs://clusterX-webhdfs/foo/bar` and `viewfs://clusterX-hftp/foo/bar`
+4. `viewfs://clusterX-webhdfs/foo/bar`
- * These are URIs respectively for accessing files via the WebHDFS file system and the HFTP file system.
+ * It is an URI for accessing files via the WebHDFS file system.
5. `http://namenodeClusterX:http_port/webhdfs/v1/foo/bar` and `http://proxyClusterX:http_port/foo/bar`
[20/50] [abbrv] hadoop git commit: YARN-4569. Remove incorrect part
of maxResources in FairScheduler documentation. (Ray Chiang via kasha)
Posted by ar...@apache.org.
YARN-4569. Remove incorrect part of maxResources in FairScheduler documentation. (Ray Chiang via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0c95b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0c95b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0c95b5f
Branch: refs/heads/HDFS-1312
Commit: a0c95b5fc4c90ee3383619156619a66dfba889f7
Parents: 1c248ea
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Feb 17 18:36:12 2016 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Feb 17 18:36:12 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop-yarn-site/src/site/markdown/FairScheduler.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c95b5f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4471912..c172054 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -211,6 +211,9 @@ Release 2.9.0 - UNRELEASED
YARN-4684. TestYarnCLI#testGetContainers failing in CN locale.
(Bibin A Chundatt via vvasudev)
+ YARN-4569. Remove incorrect part of maxResources in FairScheduler
+ documentation. (Ray Chiang via kasha)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c95b5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 26c6d22..c9c820f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -96,7 +96,7 @@ The allocation file must be in XML format. The format contains five types of ele
* minResources: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
- * maxResources: maximum resources a queue is allowed, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. A queue will never be assigned a container that would put its aggregate usage over this limit.
+ * maxResources: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit.
* maxRunningApps: limit the number of apps from the queue to run at once
[05/50] [abbrv] hadoop git commit: HDFS-9773. Remove dead code
related to SimulatedFSDataset in tests. Contributed by Brahma Reddy Battula.
Posted by ar...@apache.org.
HDFS-9773. Remove dead code related to SimulatedFSDataset in tests. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0738ae6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0738ae6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0738ae6
Branch: refs/heads/HDFS-1312
Commit: b0738ae673e4a95060b33498c049cd7790d6e371
Parents: ac5da11
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 15 15:35:06 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 15 15:35:06 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/TestFileAppend.java | 23 +-------------------
.../org/apache/hadoop/hdfs/TestFileAppend2.java | 8 -------
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 5 -----
.../org/apache/hadoop/hdfs/TestLargeBlock.java | 5 -----
.../shortcircuit/TestShortCircuitLocalRead.java | 7 ------
6 files changed, 4 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0738ae6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eb2b21e..d144ba7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1925,6 +1925,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9797. Log Standby exceptions thrown by RequestHedgingProxyProvider
at DEBUG Level (Inigo Goiri via asuresh)
+ HDFS-9773. Remove dead code related to SimulatedFSDataset in tests.
+ (Brahma Reddy Battula via aajisaka)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0738ae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index 56b6590..2768d21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
import org.apache.hadoop.io.IOUtils;
@@ -59,8 +58,6 @@ import org.junit.Test;
public class TestFileAppend{
private static final long RANDOM_TEST_RUNTIME = 10000;
- final boolean simulatedStorage = false;
-
private static byte[] fileContents = null;
//
@@ -101,13 +98,7 @@ public class TestFileAppend{
}
byte[] expected =
new byte[AppendTestUtil.NUM_BLOCKS * AppendTestUtil.BLOCK_SIZE];
- if (simulatedStorage) {
- LocatedBlocks lbs = fileSys.getClient().getLocatedBlocks(name.toString(),
- 0, AppendTestUtil.FILE_SIZE);
- DFSTestUtil.fillExpectedBuf(lbs, expected);
- } else {
- System.arraycopy(fileContents, 0, expected, 0, expected.length);
- }
+ System.arraycopy(fileContents, 0, expected, 0, expected.length);
// do a sanity check. Read the file
// do not check file status since the file is not yet closed.
AppendTestUtil.checkFullFile(fileSys, name,
@@ -118,9 +109,6 @@ public class TestFileAppend{
@Test
public void testBreakHardlinksIfNeeded() throws IOException {
Configuration conf = new HdfsConfiguration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
InetSocketAddress addr = new InetSocketAddress("localhost",
@@ -186,9 +174,6 @@ public class TestFileAppend{
@Test
public void testSimpleFlush() throws IOException {
Configuration conf = new HdfsConfiguration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
DistributedFileSystem fs = cluster.getFileSystem();
@@ -242,9 +227,6 @@ public class TestFileAppend{
@Test
public void testComplexFlush() throws IOException {
Configuration conf = new HdfsConfiguration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
DistributedFileSystem fs = cluster.getFileSystem();
@@ -293,9 +275,6 @@ public class TestFileAppend{
@Test(expected = FileNotFoundException.class)
public void testFileNotFound() throws IOException {
Configuration conf = new HdfsConfiguration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0738ae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
index 3c72db3..cd1b851 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -59,7 +58,6 @@ public class TestFileAppend2 {
}
static final int numBlocks = 5;
- final boolean simulatedStorage = false;
private byte[] fileContents = null;
@@ -81,9 +79,6 @@ public class TestFileAppend2 {
@Test
public void testSimpleAppend() throws IOException {
final Configuration conf = new HdfsConfiguration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 50);
fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
@@ -233,9 +228,6 @@ public class TestFileAppend2 {
@Test
public void testSimpleAppend2() throws Exception {
final Configuration conf = new HdfsConfiguration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 50);
fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0738ae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
index 62f6cd2..265b510 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
@@ -67,7 +66,6 @@ public class TestFileAppend4 {
MiniDFSCluster cluster;
Path file1;
FSDataOutputStream stm;
- final boolean simulatedStorage = false;
{
DFSTestUtil.setNameNodeLogLevel(Level.ALL);
@@ -78,9 +76,6 @@ public class TestFileAppend4 {
@Before
public void setUp() throws Exception {
this.conf = new Configuration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
// lower heartbeat interval for fast recognition of DN death
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0738ae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
index 0896120..a37da35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLargeBlock.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.junit.Test;
/**
@@ -51,7 +50,6 @@ public class TestLargeBlock {
// should we verify the data read back from the file? (slow)
static final boolean verifyData = true;
static final byte[] pattern = { 'D', 'E', 'A', 'D', 'B', 'E', 'E', 'F'};
- static final boolean simulatedStorage = false;
// creates a file
static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl,
@@ -177,9 +175,6 @@ public class TestLargeBlock {
final long fileSize = blockSize + 1L;
Configuration conf = new Configuration();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0738ae6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
index a069003..f4fbebc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
@@ -97,7 +97,6 @@ public class TestShortCircuitLocalRead {
static final long seed = 0xDEADBEEFL;
static final int blockSize = 5120;
- final boolean simulatedStorage = false;
// creates a file but does not close it
static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl)
@@ -268,9 +267,6 @@ public class TestShortCircuitLocalRead {
conf.setBoolean(
HdfsClientConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true);
}
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
.format(true).build();
FileSystem fs = cluster.getFileSystem();
@@ -399,9 +395,6 @@ public class TestShortCircuitLocalRead {
new File(sockDir.getDir(),
"testSkipWithVerifyChecksum._PORT.sock").getAbsolutePath());
DomainSocket.disableBindPathValidation();
- if (simulatedStorage) {
- SimulatedFSDataset.setFactory(conf);
- }
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
.format(true).build();
FileSystem fs = cluster.getFileSystem();
[46/50] [abbrv] hadoop git commit: YARN-4707. Remove the extra char
(>) from SecureContainer.md. Contributed by Brahma Reddy Battula.
Posted by ar...@apache.org.
YARN-4707. Remove the extra char (>) from SecureContainer.md. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e7d4d51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e7d4d51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e7d4d51
Branch: refs/heads/HDFS-1312
Commit: 5e7d4d51f856298e99a00895013ac665e67549b3
Parents: 342c957
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Feb 22 17:17:40 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Feb 22 17:17:40 2016 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop-yarn-site/src/site/markdown/SecureContainer.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e7d4d51/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d6aa4e4..e39917f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1521,6 +1521,9 @@ Release 2.7.3 - UNRELEASED
YARN-3602. TestResourceLocalizationService.testPublicResourceInitializesLocalDir
fails Intermittently due to IOException from cleanup. (zhihai xu via xgong)
+ YARN-4707. Remove the extra char (>) from SecureContainer.md.
+ (Brahma Reddy Battula via aajisaka)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e7d4d51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
index f32e460..cd4f913 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
@@ -30,7 +30,7 @@ Secure Containers work only in the context of secured YARN clusters.
###Linux Secure Container Executor
- On Linux environment the secure container executor is the `LinuxContainerExecutor`. It uses an external program called the **container-executor**\> to launch the container. This program has the `setuid` access right flag set which allows it to launch the container with the permissions of the YARN application user.
+ On Linux environment the secure container executor is the `LinuxContainerExecutor`. It uses an external program called the **container-executor** to launch the container. This program has the `setuid` access right flag set which allows it to launch the container with the permissions of the YARN application user.
###Configuration