You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/04/05 21:30:34 UTC
[01/50] [abbrv] hadoop git commit: MAPREDUCE-6663. [NNBench] Refactor
nnbench as a Tool implementation. Contributed by Brahma Reddy Battula.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-1312 12b4cf787 -> 48a8c9c3e
MAPREDUCE-6663. [NNBench] Refactor nnbench as a Tool implementation. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/690d8a36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/690d8a36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/690d8a36
Branch: refs/heads/HDFS-1312
Commit: 690d8a368d3e967495eafea27659b6124989c89e
Parents: 8286270
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Mar 30 11:42:54 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Mar 30 11:42:54 2016 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/NNBench.java | 239 ++++++++++---------
.../org/apache/hadoop/hdfs/TestNNBench.java | 84 +++++++
2 files changed, 211 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8a36/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
index 96c4710..ee3cc00 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
@@ -25,7 +25,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
-import java.net.InetAddress;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
@@ -33,6 +32,7 @@ import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
@@ -54,6 +55,8 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* This program executes a specified operation that applies load to
@@ -74,49 +77,48 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
* must be run before running the other operations.
*/
-public class NNBench {
+public class NNBench extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(
"org.apache.hadoop.hdfs.NNBench");
- protected static String CONTROL_DIR_NAME = "control";
- protected static String OUTPUT_DIR_NAME = "output";
- protected static String DATA_DIR_NAME = "data";
- protected static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log";
- protected static final String NNBENCH_VERSION = "NameNode Benchmark 0.4";
-
- public static String operation = "none";
- public static long numberOfMaps = 1l; // default is 1
- public static long numberOfReduces = 1l; // default is 1
- public static long startTime =
+ private static String CONTROL_DIR_NAME = "control";
+ private static String OUTPUT_DIR_NAME = "output";
+ private static String DATA_DIR_NAME = "data";
+ static final String DEFAULT_RES_FILE_NAME = "NNBench_results.log";
+ private static final String NNBENCH_VERSION = "NameNode Benchmark 0.4";
+
+ private String operation = "none";
+ private long numberOfMaps = 1l; // default is 1
+ private long numberOfReduces = 1l; // default is 1
+ private long startTime =
System.currentTimeMillis() + (120 * 1000); // default is 'now' + 2min
- public static long blockSize = 1l; // default is 1
- public static int bytesToWrite = 0; // default is 0
- public static long bytesPerChecksum = 1l; // default is 1
- public static long numberOfFiles = 1l; // default is 1
- public static short replicationFactorPerFile = 1; // default is 1
- public static String baseDir = "/benchmarks/NNBench"; // default
- public static boolean readFileAfterOpen = false; // default is to not read
-
+ private long blockSize = 1l; // default is 1
+ private int bytesToWrite = 0; // default is 0
+ private long bytesPerChecksum = 1l; // default is 1
+ private long numberOfFiles = 1l; // default is 1
+ private short replicationFactorPerFile = 1; // default is 1
+ private String baseDir = "/benchmarks/NNBench"; // default
+ private boolean readFileAfterOpen = false; // default is to not read
+ private boolean isHelpMessage = false;
// Supported operations
private static final String OP_CREATE_WRITE = "create_write";
private static final String OP_OPEN_READ = "open_read";
private static final String OP_RENAME = "rename";
private static final String OP_DELETE = "delete";
+ private static final int MAX_OPERATION_EXCEPTIONS = 1000;
// To display in the format that matches the NN and DN log format
// Example: 2007-10-26 00:01:19,853
static SimpleDateFormat sdf =
new SimpleDateFormat("yyyy-MM-dd' 'HH:mm:ss','S");
-
- private static Configuration config = new Configuration();
/**
* Clean up the files before a test run
*
* @throws IOException on error
*/
- private static void cleanupBeforeTestrun() throws IOException {
- FileSystem tempFS = FileSystem.get(config);
+ private void cleanupBeforeTestrun() throws IOException {
+ FileSystem tempFS = FileSystem.get(getConf());
// Delete the data directory only if it is the create/write operation
if (operation.equals(OP_CREATE_WRITE)) {
@@ -133,8 +135,7 @@ public class NNBench {
*
* @throws IOException on error
*/
- private static void createControlFiles() throws IOException {
- FileSystem tempFS = FileSystem.get(config);
+ private void createControlFiles() throws IOException {
LOG.info("Creating " + numberOfMaps + " control files");
for (int i = 0; i < numberOfMaps; i++) {
@@ -144,8 +145,9 @@ public class NNBench {
SequenceFile.Writer writer = null;
try {
- writer = SequenceFile.createWriter(tempFS, config, filePath, Text.class,
- LongWritable.class, CompressionType.NONE);
+ writer = SequenceFile.createWriter(getConf(), Writer.file(filePath),
+ Writer.keyClass(Text.class), Writer.valueClass(LongWritable.class),
+ Writer.compression(CompressionType.NONE));
writer.append(new Text(strFileName), new LongWritable(i));
} finally {
if (writer != null) {
@@ -208,23 +210,23 @@ public class NNBench {
* line's arguments
* @param length total number of arguments
*/
- public static void checkArgs(final int index, final int length) {
+ private static void checkArgs(final int index, final int length) {
if (index == length) {
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException("Not enough arguments");
}
}
/**
* Parse input arguments
+ * @param args array of command line's parameters to be parsed
*
- * @param args array of command line's parameters to be parsed
*/
- public static void parseInputs(final String[] args) {
+ private void parseInputs(final String[] args) {
// If there are no command line arguments, exit
if (args.length == 0) {
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException("Give valid inputs");
}
// Parse command line args
@@ -263,7 +265,7 @@ public class NNBench {
readFileAfterOpen = Boolean.parseBoolean(args[++i]);
} else if (args[i].equals("-help")) {
displayUsage();
- System.exit(-1);
+ isHelpMessage = true;
}
}
@@ -281,31 +283,30 @@ public class NNBench {
LOG.info(" Read file after open: " + readFileAfterOpen);
// Set user-defined parameters, so the map method can access the values
- config.set("test.nnbench.operation", operation);
- config.setLong("test.nnbench.maps", numberOfMaps);
- config.setLong("test.nnbench.reduces", numberOfReduces);
- config.setLong("test.nnbench.starttime", startTime);
- config.setLong("test.nnbench.blocksize", blockSize);
- config.setInt("test.nnbench.bytestowrite", bytesToWrite);
- config.setLong("test.nnbench.bytesperchecksum", bytesPerChecksum);
- config.setLong("test.nnbench.numberoffiles", numberOfFiles);
- config.setInt("test.nnbench.replicationfactor",
+ getConf().set("test.nnbench.operation", operation);
+ getConf().setLong("test.nnbench.maps", numberOfMaps);
+ getConf().setLong("test.nnbench.reduces", numberOfReduces);
+ getConf().setLong("test.nnbench.starttime", startTime);
+ getConf().setLong("test.nnbench.blocksize", blockSize);
+ getConf().setInt("test.nnbench.bytestowrite", bytesToWrite);
+ getConf().setLong("test.nnbench.bytesperchecksum", bytesPerChecksum);
+ getConf().setLong("test.nnbench.numberoffiles", numberOfFiles);
+ getConf().setInt("test.nnbench.replicationfactor",
(int) replicationFactorPerFile);
- config.set("test.nnbench.basedir", baseDir);
- config.setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen);
+ getConf().set("test.nnbench.basedir", baseDir);
+ getConf().setBoolean("test.nnbench.readFileAfterOpen", readFileAfterOpen);
- config.set("test.nnbench.datadir.name", DATA_DIR_NAME);
- config.set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME);
- config.set("test.nnbench.controldir.name", CONTROL_DIR_NAME);
+ getConf().set("test.nnbench.datadir.name", DATA_DIR_NAME);
+ getConf().set("test.nnbench.outputdir.name", OUTPUT_DIR_NAME);
+ getConf().set("test.nnbench.controldir.name", CONTROL_DIR_NAME);
}
/**
* Analyze the results
- *
* @throws IOException on error
*/
- private static void analyzeResults() throws IOException {
- final FileSystem fs = FileSystem.get(config);
+ private int analyzeResults() throws IOException {
+ final FileSystem fs = FileSystem.get(getConf());
Path reduceDir = new Path(baseDir, OUTPUT_DIR_NAME);
long totalTimeAL1 = 0l;
@@ -322,32 +323,31 @@ public class NNBench {
for (FileStatus status : fss) {
Path reduceFile = status.getPath();
- DataInputStream in;
- in = new DataInputStream(fs.open(reduceFile));
-
- BufferedReader lines;
- lines = new BufferedReader(new InputStreamReader(in));
-
- String line;
- while ((line = lines.readLine()) != null) {
- StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;");
- String attr = tokens.nextToken();
- if (attr.endsWith(":totalTimeAL1")) {
- totalTimeAL1 = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":totalTimeAL2")) {
- totalTimeAL2 = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":totalTimeTPmS")) {
- totalTimeTPmS = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":latemaps")) {
- lateMaps = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":numOfExceptions")) {
- numOfExceptions = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":successfulFileOps")) {
- successfulFileOps = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":mapStartTimeTPmS")) {
- mapStartTimeTPmS = Long.parseLong(tokens.nextToken());
- } else if (attr.endsWith(":mapEndTimeTPmS")) {
- mapEndTimeTPmS = Long.parseLong(tokens.nextToken());
+ try (DataInputStream in = new DataInputStream(fs.open(reduceFile));
+ BufferedReader lines =
+ new BufferedReader(new InputStreamReader(in))) {
+
+ String line;
+ while ((line = lines.readLine()) != null) {
+ StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%;");
+ String attr = tokens.nextToken();
+ if (attr.endsWith(":totalTimeAL1")) {
+ totalTimeAL1 = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":totalTimeAL2")) {
+ totalTimeAL2 = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":totalTimeTPmS")) {
+ totalTimeTPmS = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":latemaps")) {
+ lateMaps = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":numOfExceptions")) {
+ numOfExceptions = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":successfulFileOps")) {
+ successfulFileOps = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":mapStartTimeTPmS")) {
+ mapStartTimeTPmS = Long.parseLong(tokens.nextToken());
+ } else if (attr.endsWith(":mapEndTimeTPmS")) {
+ mapEndTimeTPmS = Long.parseLong(tokens.nextToken());
+ }
}
}
}
@@ -444,25 +444,29 @@ public class NNBench {
" RAW DATA: # of exceptions: " + numOfExceptions,
"" };
- PrintStream res = new PrintStream(new FileOutputStream(
- new File(DEFAULT_RES_FILE_NAME), true));
-
- // Write to a file and also dump to log
- for(int i = 0; i < resultLines.length; i++) {
- LOG.info(resultLines[i]);
- res.println(resultLines[i]);
+ try (PrintStream res = new PrintStream(
+ new FileOutputStream(new File(DEFAULT_RES_FILE_NAME), true))) {
+ // Write to a file and also dump to log
+ for (String resultLine : resultLines) {
+ LOG.info(resultLine);
+ res.println(resultLine);
+ }
+ }
+ if(numOfExceptions >= MAX_OPERATION_EXCEPTIONS){
+ return -1;
}
+ return 0;
}
-
+
/**
* Run the test
*
* @throws IOException on error
*/
- public static void runTests() throws IOException {
- config.setLong("io.bytes.per.checksum", bytesPerChecksum);
+ private void runTests() throws IOException {
+ getConf().setLong("io.bytes.per.checksum", bytesPerChecksum);
- JobConf job = new JobConf(config, NNBench.class);
+ JobConf job = new JobConf(getConf(), NNBench.class);
job.setJobName("NNBench-" + operation);
FileInputFormat.setInputPaths(job, new Path(baseDir, CONTROL_DIR_NAME));
@@ -487,7 +491,7 @@ public class NNBench {
/**
* Validate the inputs
*/
- public static void validateInputs() {
+ private void validateInputs() {
// If it is not one of the four operations, then fail
if (!operation.equals(OP_CREATE_WRITE) &&
!operation.equals(OP_OPEN_READ) &&
@@ -495,7 +499,8 @@ public class NNBench {
!operation.equals(OP_DELETE)) {
System.err.println("Error: Unknown operation: " + operation);
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Unknown operation: " + operation);
}
// If number of maps is a negative number, then fail
@@ -503,57 +508,66 @@ public class NNBench {
if (numberOfMaps < 0) {
System.err.println("Error: Number of maps must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Number of maps must be a positive number");
}
// If number of reduces is a negative number or 0, then fail
if (numberOfReduces <= 0) {
System.err.println("Error: Number of reduces must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Number of reduces must be a positive number");
}
// If blocksize is a negative number or 0, then fail
if (blockSize <= 0) {
System.err.println("Error: Block size must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Block size must be a positive number");
}
// If bytes to write is a negative number, then fail
if (bytesToWrite < 0) {
System.err.println("Error: Bytes to write must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Bytes to write must be a positive number");
}
// If bytes per checksum is a negative number, then fail
if (bytesPerChecksum < 0) {
System.err.println("Error: Bytes per checksum must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Bytes per checksum must be a positive number");
}
// If number of files is a negative number, then fail
if (numberOfFiles < 0) {
System.err.println("Error: Number of files must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Number of files must be a positive number");
}
// If replication factor is a negative number, then fail
if (replicationFactorPerFile < 0) {
System.err.println("Error: Replication factor must be a positive number");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Replication factor must be a positive number");
}
// If block size is not a multiple of bytesperchecksum, fail
if (blockSize % bytesPerChecksum != 0) {
- System.err.println("Error: Block Size in bytes must be a multiple of " +
- "bytes per checksum: ");
+ System.err.println("Error: Block Size in bytes must be a multiple of "
+ + "bytes per checksum: ");
displayUsage();
- System.exit(-1);
+ throw new HadoopIllegalArgumentException(
+ "Error: Block Size in bytes must be a multiple of "
+ + "bytes per checksum:");
}
}
/**
@@ -562,13 +576,22 @@ public class NNBench {
* @param args array of command line arguments
* @throws IOException indicates a problem with test startup
*/
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new NNBench(), args);
+ System.exit(res);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
// Display the application version string
displayVersion();
// Parse the inputs
parseInputs(args);
-
+ if (isHelpMessage) {
+ return 0;
+ }
+
// Validate inputs
validateInputs();
@@ -582,7 +605,7 @@ public class NNBench {
runTests();
// Analyze results
- analyzeResults();
+ return analyzeResults();
}
@@ -592,7 +615,6 @@ public class NNBench {
static class NNBenchMapper extends Configured
implements Mapper<Text, LongWritable, Text, Text> {
FileSystem filesystem = null;
- private String hostName = null;
long numberOfFiles = 1l;
long blkSize = 1l;
@@ -602,7 +624,6 @@ public class NNBench {
String dataDirName = null;
String op = null;
boolean readFile = false;
- final int MAX_OPERATION_EXCEPTIONS = 1000;
// Data to collect from the operation
int numOfExceptions = 0;
@@ -628,12 +649,6 @@ public class NNBench {
} catch(Exception e) {
throw new RuntimeException("Cannot get file system.", e);
}
-
- try {
- hostName = InetAddress.getLocalHost().getHostName();
- } catch(Exception e) {
- throw new RuntimeException("Error getting hostname", e);
- }
}
/**
@@ -678,7 +693,7 @@ public class NNBench {
LongWritable value,
OutputCollector<Text, Text> output,
Reporter reporter) throws IOException {
- Configuration conf = filesystem.getConf();
+ Configuration conf = getConf();
numberOfFiles = conf.getLong("test.nnbench.numberoffiles", 1l);
blkSize = conf.getLong("test.nnbench.blocksize", 1l);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8a36/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java
new file mode 100644
index 0000000..9f9814d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/TestNNBench.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.HadoopTestCase;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Test;
+
+public class TestNNBench extends HadoopTestCase {
+ private static final String BASE_DIR =
+ new File(System.getProperty("test.build.data", "build/test/data"),
+ "NNBench").getAbsolutePath();
+
+ public TestNNBench() throws IOException {
+ super(LOCAL_MR, LOCAL_FS, 1, 1);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ getFileSystem().delete(new Path(BASE_DIR), true);
+ getFileSystem().delete(new Path(NNBench.DEFAULT_RES_FILE_NAME), true);
+ super.tearDown();
+ }
+
+ @Test(timeout = 30000)
+ public void testNNBenchCreateReadAndDelete() throws Exception {
+ runNNBench(createJobConf(), "create_write");
+ Path path = new Path(BASE_DIR + "/data/file_0_0");
+ assertTrue("create_write should create the file",
+ getFileSystem().exists(path));
+ runNNBench(createJobConf(), "open_read");
+ runNNBench(createJobConf(), "delete");
+ assertFalse("Delete operation should delete the file",
+ getFileSystem().exists(path));
+ }
+
+ @Test(timeout = 30000)
+ public void testNNBenchCreateAndRename() throws Exception {
+ runNNBench(createJobConf(), "create_write");
+ Path path = new Path(BASE_DIR + "/data/file_0_0");
+ assertTrue("create_write should create the file",
+ getFileSystem().exists(path));
+ runNNBench(createJobConf(), "rename");
+ Path renamedPath = new Path(BASE_DIR + "/data/file_0_r_0");
+ assertFalse("Rename should rename the file", getFileSystem().exists(path));
+ assertTrue("Rename should rename the file",
+ getFileSystem().exists(renamedPath));
+ }
+
+ private void runNNBench(Configuration conf, String operation)
+ throws Exception {
+ String[] genArgs = { "-operation", operation, "-baseDir", BASE_DIR,
+ "-startTime", "" + (Time.now() / 1000 + 3) };
+
+ assertEquals(0, ToolRunner.run(conf, new NNBench(), genArgs));
+ }
+
+}
[30/50] [abbrv] hadoop git commit: YARN-4607. Pagination support for
AppAttempt page TotalOutstandingResource Requests table. Contributed by Bibin
A Chundatt
Posted by ae...@apache.org.
YARN-4607. Pagination support for AppAttempt page TotalOutstandingResource Requests table. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e6f9297
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e6f9297
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e6f9297
Branch: refs/heads/HDFS-1312
Commit: 1e6f92977dc5431b117745feb5a3491e88a559c0
Parents: 0ecdd4c
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Apr 4 08:09:29 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Apr 4 08:09:29 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/server/webapp/WebPageUtils.java | 6 ++
.../resourcemanager/webapp/AppAttemptPage.java | 4 +-
.../webapp/RMAppAttemptBlock.java | 61 ++++++++++++--------
3 files changed, 45 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e6f9297/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index a07baa2..3a26ae5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -95,4 +95,10 @@ public class WebPageUtils {
.append(", 'mRender': parseHadoopID }]").toString();
}
+ public static String resourceRequestsTableInit() {
+ return tableInit().append(", 'aaData': resourceRequestsTableData")
+ .append(", bDeferRender: true").append(", bProcessing: true}")
+ .toString();
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e6f9297/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
index df5fb9e..45f1887 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
@@ -41,8 +41,10 @@ public class AppAttemptPage extends RmView {
: join("Application Attempt ",
$(YarnWebParams.APPLICATION_ATTEMPT_ID)));
- set(DATATABLES_ID, "containers");
+ set(DATATABLES_ID, "containers resourceRequests");
set(initID(DATATABLES, "containers"), WebPageUtils.containersTableInit());
+ set(initID(DATATABLES, "resourceRequests"),
+ WebPageUtils.resourceRequestsTableInit());
setTableStyles(html, "containers", ".queue {width:6em}", ".ui {width:8em}");
set(YarnWebParams.WEB_UI_TYPE, YarnWebParams.RM_WEB_UI);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e6f9297/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 668269e..6fef367 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -26,6 +26,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.util.Collection;
import java.util.List;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -46,6 +47,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
@@ -75,35 +77,44 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
}
DIV<Hamlet> div = html.div(_INFO_WRAP);
- TABLE<DIV<Hamlet>> table =
- div.h3("Total Outstanding Resource Requests: "
- + getTotalResource(resourceRequests)).table(
- "#ResourceRequests");
-
- table.tr().
- th(_TH, "Priority").
- th(_TH, "ResourceName").
- th(_TH, "Capability").
- th(_TH, "NumContainers").
- th(_TH, "RelaxLocality").
- th(_TH, "NodeLabelExpression").
- _();
+ // Requests Table
+ TBODY<TABLE<DIV<Hamlet>>> tbody = div
+ .h3("Total Outstanding Resource Requests: "
+ + getTotalResource(resourceRequests))
+ .table("#resourceRequests").thead().tr().th(".priority", "Priority")
+ .th(".resource", "ResourceName").th(".capacity", "Capability")
+ .th(".containers", "NumContainers")
+ .th(".relaxlocality", "RelaxLocality")
+ .th(".labelexpression", "NodeLabelExpression")._()._().tbody();
- boolean odd = false;
- for (ResourceRequest request : resourceRequests) {
- if (request.getNumContainers() == 0) {
+ StringBuilder resourceRequestTableData = new StringBuilder("[\n");
+ for (ResourceRequest resourceRequest : resourceRequests) {
+ if (resourceRequest.getNumContainers() == 0) {
continue;
}
- table.tr((odd = !odd) ? _ODD : _EVEN)
- .td(String.valueOf(request.getPriority()))
- .td(request.getResourceName())
- .td(String.valueOf(request.getCapability()))
- .td(String.valueOf(request.getNumContainers()))
- .td(String.valueOf(request.getRelaxLocality()))
- .td(request.getNodeLabelExpression() == null ? "N/A" : request
- .getNodeLabelExpression())._();
+ resourceRequestTableData.append("[\"")
+ .append(String.valueOf(resourceRequest.getPriority())).append("\",\"")
+ .append(resourceRequest.getResourceName()).append("\",\"")
+ .append(StringEscapeUtils.escapeJavaScript(StringEscapeUtils
+ .escapeHtml(String.valueOf(resourceRequest.getCapability()))))
+ .append("\",\"")
+ .append(String.valueOf(resourceRequest.getNumContainers()))
+ .append("\",\"")
+ .append(String.valueOf(resourceRequest.getRelaxLocality()))
+ .append("\",\"")
+ .append(resourceRequest.getNodeLabelExpression() == null ? "N/A"
+ : resourceRequest.getNodeLabelExpression())
+ .append("\"],\n");
}
- table._();
+ if (resourceRequestTableData
+ .charAt(resourceRequestTableData.length() - 2) == ',') {
+ resourceRequestTableData.delete(resourceRequestTableData.length() - 2,
+ resourceRequestTableData.length() - 1);
+ }
+ resourceRequestTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var resourceRequestsTableData=" + resourceRequestTableData)._();
+ tbody._()._();
div._();
}
[02/50] [abbrv] hadoop git commit: HDFS-5177. blocksScheduled count
should be decremented for abandoned blocks (Contributed by Vinayakumar B)
Posted by ae...@apache.org.
HDFS-5177. blocksScheduled count should be decremented for abandoned blocks (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09d63d5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09d63d5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09d63d5a
Branch: refs/heads/HDFS-1312
Commit: 09d63d5a192b5d6b172f94ff6c94da348fd49ea6
Parents: 690d8a3
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Mar 30 14:22:00 2016 +0800
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Mar 30 14:22:00 2016 +0800
----------------------------------------------------------------------
.../blockmanagement/DatanodeStorageInfo.java | 10 ++++
.../hdfs/server/namenode/FSDirWriteFileOp.java | 4 ++
.../hadoop/hdfs/TestBlocksScheduledCounter.java | 51 ++++++++++++++++++++
3 files changed, 65 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d63d5a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c4729ea..843a8d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -298,6 +298,16 @@ public class DatanodeStorageInfo {
}
}
+ /**
+ * Decrement the number of blocks scheduled for each given storage. This will
+ * be called during abandon block or delete of UC block.
+ */
+ public static void decrementBlocksScheduled(DatanodeStorageInfo... storages) {
+ for (DatanodeStorageInfo s : storages) {
+ s.getDatanodeDescriptor().decrementBlocksScheduled(s.getStorageType());
+ }
+ }
+
@Override
public boolean equals(Object obj) {
if (this == obj) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d63d5a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 41fd869..ab2f0fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -82,6 +82,10 @@ class FSDirWriteFileOp {
if (uc == null) {
return false;
}
+ if (uc.getUnderConstructionFeature() != null) {
+ DatanodeStorageInfo.decrementBlocksScheduled(uc
+ .getUnderConstructionFeature().getExpectedStorageLocations());
+ }
fsd.getBlockManager().removeBlockFromMap(uc);
if(NameNode.stateChangeLog.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09d63d5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
index b943219..1894278 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
@@ -78,4 +78,55 @@ public class TestBlocksScheduledCounter {
out.close();
assertEquals(0, dn.getBlocksScheduled());
}
+
+ /**
+ * Abandon block should decrement the scheduledBlocks count for the dataNode.
+ */
+ @Test
+ public void testScheduledBlocksCounterShouldDecrementOnAbandonBlock()
+ throws Exception {
+ cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).numDataNodes(
+ 2).build();
+
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+
+ DatanodeManager datanodeManager = cluster.getNamesystem().getBlockManager()
+ .getDatanodeManager();
+ ArrayList<DatanodeDescriptor> dnList = new ArrayList<DatanodeDescriptor>();
+ datanodeManager.fetchDatanodes(dnList, dnList, false);
+ for (DatanodeDescriptor descriptor : dnList) {
+ assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(),
+ 0, descriptor.getBlocksScheduled());
+ }
+
+ cluster.getDataNodes().get(0).shutdown();
+ // open a file an write a few bytes:
+ FSDataOutputStream out = fs.create(new Path("/testBlockScheduledCounter"),
+ (short) 2);
+ for (int i = 0; i < 1024; i++) {
+ out.write(i);
+ }
+ // flush to make sure a block is allocated.
+ out.hflush();
+
+ DatanodeDescriptor abandonedDn = datanodeManager.getDatanode(cluster
+ .getDataNodes().get(0).getDatanodeId());
+ assertEquals("for the abandoned dn scheduled counts should be 0", 0,
+ abandonedDn.getBlocksScheduled());
+
+ for (DatanodeDescriptor descriptor : dnList) {
+ if (descriptor.equals(abandonedDn)) {
+ continue;
+ }
+ assertEquals("Blocks scheduled should be 1 for " + descriptor.getName(),
+ 1, descriptor.getBlocksScheduled());
+ }
+ // close the file and the counter should go to zero.
+ out.close();
+ for (DatanodeDescriptor descriptor : dnList) {
+ assertEquals("Blocks scheduled should be 0 for " + descriptor.getName(),
+ 0, descriptor.getBlocksScheduled());
+ }
+ }
}
[35/50] [abbrv] hadoop git commit: HDFS-9599.
TestDecommissioningStatus.testDecommissionStatus occasionally fails (Lin
Yiqun via iwasakims)
Posted by ae...@apache.org.
HDFS-9599. TestDecommissioningStatus.testDecommissionStatus occasionally fails (Lin Yiqun via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/154d2532
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/154d2532
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/154d2532
Branch: refs/heads/HDFS-1312
Commit: 154d2532cf015e9ab9141864bd3ab0d6100ef597
Parents: 7280550
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Tue Apr 5 03:19:48 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Tue Apr 5 03:19:48 2016 +0900
----------------------------------------------------------------------
.../hdfs/server/namenode/TestDecommissioningStatus.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/154d2532/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
index 789ee6f..1e7312a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
@@ -57,8 +57,8 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
/**
@@ -78,8 +78,8 @@ public class TestDecommissioningStatus {
final ArrayList<String> decommissionedNodes = new ArrayList<String>(numDatanodes);
- @BeforeClass
- public static void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
false);
@@ -113,8 +113,8 @@ public class TestDecommissioningStatus {
Logger.getLogger(DecommissionManager.class).setLevel(Level.DEBUG);
}
- @AfterClass
- public static void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
if (localFileSys != null ) cleanupFile(localFileSys, dir);
if(fileSys != null) fileSys.close();
if(cluster != null) cluster.shutdown();
[03/50] [abbrv] hadoop git commit: YARN-4822. Refactor existing
Preemption Policy of CS for easier adding new approach to select preemption
candidates. Contributed by Wangda Tan
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index 21ea495..b266665 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -18,29 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MONITORING_INTERVAL;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.isA;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -81,6 +58,25 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
private static final Log LOG =
LogFactory.getLog(TestProportionalCapacityPreemptionPolicyForNodePartitions.class);
@@ -94,8 +90,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
private ResourceCalculator rc = new DefaultResourceCalculator();
private Clock mClock = null;
- private Configuration conf = null;
- private CapacitySchedulerConfiguration csConf = null;
+ private CapacitySchedulerConfiguration conf = null;
private CapacityScheduler cs = null;
private EventHandler<SchedulerEvent> mDisp = null;
private ProportionalCapacityPreemptionPolicy policy = null;
@@ -107,24 +102,23 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
org.apache.log4j.Logger.getRootLogger().setLevel(
org.apache.log4j.Level.DEBUG);
- conf = new Configuration(false);
- conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
- conf.setLong(MONITORING_INTERVAL, 3000);
+ conf = new CapacitySchedulerConfiguration(new Configuration(false));
+ conf.setLong(
+ CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
+ conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL,
+ 3000);
// report "ideal" preempt
- conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
- conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
- conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
- ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
- conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
- // FairScheduler doesn't support this test,
- // Set CapacityScheduler as the scheduler for this test.
- conf.set("yarn.resourcemanager.scheduler.class",
- CapacityScheduler.class.getName());
+ conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND,
+ (float) 1.0);
+ conf.setFloat(
+ CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR,
+ (float) 1.0);
mClock = mock(Clock.class);
cs = mock(CapacityScheduler.class);
when(cs.getResourceCalculator()).thenReturn(rc);
when(cs.getPreemptionManager()).thenReturn(new PreemptionManager());
+ when(cs.getConfiguration()).thenReturn(conf);
nlm = mock(RMNodeLabelsManager.class);
mDisp = mock(EventHandler.class);
@@ -134,11 +128,9 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
Dispatcher disp = mock(Dispatcher.class);
when(rmContext.getDispatcher()).thenReturn(disp);
when(disp.getEventHandler()).thenReturn(mDisp);
- csConf = new CapacitySchedulerConfiguration();
- when(cs.getConfiguration()).thenReturn(csConf);
when(cs.getRMContext()).thenReturn(rmContext);
- policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs, mClock);
+ policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
partitionToResource = new HashMap<>();
nodeIdToSchedulerNodes = new HashMap<>();
nameToCSQueues = new HashMap<>();
@@ -576,7 +568,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
"c\t" // app3 in c
+ "(1,1,n1,x,20,false)"; // 20x in n1
- csConf.setPreemptionDisabled("root.b", true);
+ conf.setPreemptionDisabled("root.b", true);
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();
@@ -901,7 +893,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
when(cs.getClusterResource()).thenReturn(clusterResource);
mockApplications(appsConfig);
- policy = new ProportionalCapacityPreemptionPolicy(conf, rmContext, cs,
+ policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs,
mClock);
}
@@ -1235,7 +1227,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions {
// Setup preemption disabled
when(queue.getPreemptionDisabled()).thenReturn(
- csConf.getPreemptionDisabled(queuePath, false));
+ conf.getPreemptionDisabled(queuePath, false));
nameToCSQueues.put(queueName, queue);
when(cs.getQueue(eq(queueName))).thenReturn(queue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java
index bea7797..216ebab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPreemption.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
@@ -81,14 +82,15 @@ public class TestCapacitySchedulerPreemption {
conf = TestUtils.getConfigurationWithMultipleQueues(this.conf);
// Set preemption related configurations
- conf.setInt(ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL,
+ conf.setInt(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL,
0);
conf.setBoolean(CapacitySchedulerConfiguration.LAZY_PREEMPTION_ENALBED,
true);
+ conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND,
+ 1.0f);
conf.setFloat(
- ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND, 1.0f);
- conf.setFloat(
- ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR, 1.0f);
+ CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR,
+ 1.0f);
mgr = new NullRMNodeLabelsManager();
mgr.init(this.conf);
clock = mock(Clock.class);
@@ -484,6 +486,10 @@ public class TestCapacitySchedulerPreemption {
.isEmpty());
}
+ /*
+ * Ignore this test now because it could be a premature optimization
+ */
+ @Ignore
@Test (timeout = 60000)
public void testPreemptionPolicyCleanupKillableContainersWhenNoPreemptionNeeded()
throws Exception {
[39/50] [abbrv] hadoop git commit: HDFS-8496. Calling stopWriter()
with FSDatasetImpl lock held may block other threads (cmccabe)
Posted by ae...@apache.org.
HDFS-8496. Calling stopWriter() with FSDatasetImpl lock held may block other threads (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6b1a818
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6b1a818
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6b1a818
Branch: refs/heads/HDFS-1312
Commit: f6b1a818124cc42688c4c5acaf537d96cf00e43b
Parents: f65f5b1
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Apr 4 18:00:26 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Apr 4 18:02:15 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/ReplicaInPipeline.java | 54 ++++---
.../datanode/fsdataset/impl/FsDatasetImpl.java | 145 +++++++++++++------
.../datanode/fsdataset/impl/ReplicaMap.java | 2 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 137 ++++++++++++++++--
4 files changed, 257 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b1a818/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
index 5caca15..7326846 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
@@ -22,6 +22,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -44,7 +45,7 @@ public class ReplicaInPipeline extends ReplicaInfo
private long bytesAcked;
private long bytesOnDisk;
private byte[] lastChecksum;
- private Thread writer;
+ private AtomicReference<Thread> writer = new AtomicReference<Thread>();
/**
* Bytes reserved for this replica on the containing volume.
@@ -97,7 +98,7 @@ public class ReplicaInPipeline extends ReplicaInfo
super( blockId, len, genStamp, vol, dir);
this.bytesAcked = len;
this.bytesOnDisk = len;
- this.writer = writer;
+ this.writer.set(writer);
this.bytesReserved = bytesToReserve;
this.originalBytesReserved = bytesToReserve;
}
@@ -110,7 +111,7 @@ public class ReplicaInPipeline extends ReplicaInfo
super(from);
this.bytesAcked = from.getBytesAcked();
this.bytesOnDisk = from.getBytesOnDisk();
- this.writer = from.writer;
+ this.writer.set(from.writer.get());
this.bytesReserved = from.bytesReserved;
this.originalBytesReserved = from.originalBytesReserved;
}
@@ -175,18 +176,11 @@ public class ReplicaInPipeline extends ReplicaInfo
return new ChunkChecksum(getBytesOnDisk(), lastChecksum);
}
- /**
- * Set the thread that is writing to this replica
- * @param writer a thread writing to this replica
- */
- public void setWriter(Thread writer) {
- this.writer = writer;
- }
-
public void interruptThread() {
- if (writer != null && writer != Thread.currentThread()
- && writer.isAlive()) {
- this.writer.interrupt();
+ Thread thread = writer.get();
+ if (thread != null && thread != Thread.currentThread()
+ && thread.isAlive()) {
+ thread.interrupt();
}
}
@@ -196,17 +190,35 @@ public class ReplicaInPipeline extends ReplicaInfo
}
/**
+ * Attempt to set the writer to a new value.
+ */
+ public boolean attemptToSetWriter(Thread prevWriter, Thread newWriter) {
+ return writer.compareAndSet(prevWriter, newWriter);
+ }
+
+ /**
* Interrupt the writing thread and wait until it dies
* @throws IOException the waiting is interrupted
*/
public void stopWriter(long xceiverStopTimeout) throws IOException {
- if (writer != null && writer != Thread.currentThread() && writer.isAlive()) {
- writer.interrupt();
+ while (true) {
+ Thread thread = writer.get();
+ if ((thread == null) || (thread == Thread.currentThread()) ||
+ (!thread.isAlive())) {
+ if (writer.compareAndSet(thread, null) == true) {
+ return; // Done
+ }
+ // The writer changed. Go back to the start of the loop and attempt to
+ // stop the new writer.
+ continue;
+ }
+ thread.interrupt();
try {
- writer.join(xceiverStopTimeout);
- if (writer.isAlive()) {
- final String msg = "Join on writer thread " + writer + " timed out";
- DataNode.LOG.warn(msg + "\n" + StringUtils.getStackTrace(writer));
+ thread.join(xceiverStopTimeout);
+ if (thread.isAlive()) {
+ // Our thread join timed out.
+ final String msg = "Join on writer thread " + thread + " timed out";
+ DataNode.LOG.warn(msg + "\n" + StringUtils.getStackTrace(thread));
throw new IOException(msg);
}
} catch (InterruptedException e) {
@@ -214,7 +226,7 @@ public class ReplicaInPipeline extends ReplicaInfo
}
}
}
-
+
@Override // Object
public int hashCode() {
return super.hashCode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b1a818/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index d6a0df6..240345c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1207,8 +1207,20 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
return newReplicaInfo;
}
+ private static class MustStopExistingWriter extends Exception {
+ private final ReplicaInPipeline rip;
+
+ MustStopExistingWriter(ReplicaInPipeline rip) {
+ this.rip = rip;
+ }
+
+ ReplicaInPipeline getReplica() {
+ return rip;
+ }
+ }
+
private ReplicaInfo recoverCheck(ExtendedBlock b, long newGS,
- long expectedBlockLen) throws IOException {
+ long expectedBlockLen) throws IOException, MustStopExistingWriter {
ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
// check state
@@ -1232,9 +1244,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
long replicaLen = replicaInfo.getNumBytes();
if (replicaInfo.getState() == ReplicaState.RBW) {
ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo;
- // kill the previous writer
- rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout());
- rbw.setWriter(Thread.currentThread());
+ if (!rbw.attemptToSetWriter(null, Thread.currentThread())) {
+ throw new MustStopExistingWriter(rbw);
+ }
// check length: bytesRcvd, bytesOnDisk, and bytesAcked should be the same
if (replicaLen != rbw.getBytesOnDisk()
|| replicaLen != rbw.getBytesAcked()) {
@@ -1260,39 +1272,55 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException {
LOG.info("Recover failed append to " + b);
- ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
+ while (true) {
+ try {
+ synchronized (this) {
+ ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
- FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
- ReplicaBeingWritten replica;
- try {
- // change the replica's state/gs etc.
- if (replicaInfo.getState() == ReplicaState.FINALIZED) {
- replica = append(b.getBlockPoolId(), (FinalizedReplica) replicaInfo,
- newGS, b.getNumBytes());
- } else { //RBW
- bumpReplicaGS(replicaInfo, newGS);
- replica = (ReplicaBeingWritten) replicaInfo;
+ FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
+ ReplicaBeingWritten replica;
+ try {
+ // change the replica's state/gs etc.
+ if (replicaInfo.getState() == ReplicaState.FINALIZED) {
+ replica = append(b.getBlockPoolId(), (FinalizedReplica) replicaInfo,
+ newGS, b.getNumBytes());
+ } else { //RBW
+ bumpReplicaGS(replicaInfo, newGS);
+ replica = (ReplicaBeingWritten) replicaInfo;
+ }
+ } catch (IOException e) {
+ IOUtils.cleanup(null, ref);
+ throw e;
+ }
+ return new ReplicaHandler(replica, ref);
+ }
+ } catch (MustStopExistingWriter e) {
+ e.getReplica().stopWriter(datanode.getDnConf().getXceiverStopTimeout());
}
- } catch (IOException e) {
- IOUtils.cleanup(null, ref);
- throw e;
}
- return new ReplicaHandler(replica, ref);
}
@Override // FsDatasetSpi
public synchronized Replica recoverClose(ExtendedBlock b, long newGS,
long expectedBlockLen) throws IOException {
LOG.info("Recover failed close " + b);
- // check replica's state
- ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
- // bump the replica's GS
- bumpReplicaGS(replicaInfo, newGS);
- // finalize the replica if RBW
- if (replicaInfo.getState() == ReplicaState.RBW) {
- finalizeReplica(b.getBlockPoolId(), replicaInfo);
+ while (true) {
+ try {
+ synchronized (this) {
+ // check replica's state
+ ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
+ // bump the replica's GS
+ bumpReplicaGS(replicaInfo, newGS);
+ // finalize the replica if RBW
+ if (replicaInfo.getState() == ReplicaState.RBW) {
+ finalizeReplica(b.getBlockPoolId(), replicaInfo);
+ }
+ return replicaInfo;
+ }
+ } catch (MustStopExistingWriter e) {
+ e.getReplica().stopWriter(datanode.getDnConf().getXceiverStopTimeout());
+ }
}
- return replicaInfo;
}
/**
@@ -1384,26 +1412,37 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public synchronized ReplicaHandler recoverRbw(
+ public ReplicaHandler recoverRbw(
ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
throws IOException {
LOG.info("Recover RBW replica " + b);
- ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
-
- // check the replica's state
- if (replicaInfo.getState() != ReplicaState.RBW) {
- throw new ReplicaNotFoundException(
- ReplicaNotFoundException.NON_RBW_REPLICA + replicaInfo);
+ while (true) {
+ try {
+ synchronized (this) {
+ ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
+
+ // check the replica's state
+ if (replicaInfo.getState() != ReplicaState.RBW) {
+ throw new ReplicaNotFoundException(
+ ReplicaNotFoundException.NON_RBW_REPLICA + replicaInfo);
+ }
+ ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo;
+ if (!rbw.attemptToSetWriter(null, Thread.currentThread())) {
+ throw new MustStopExistingWriter(rbw);
+ }
+ LOG.info("Recovering " + rbw);
+ return recoverRbwImpl(rbw, b, newGS, minBytesRcvd, maxBytesRcvd);
+ }
+ } catch (MustStopExistingWriter e) {
+ e.getReplica().stopWriter(datanode.getDnConf().getXceiverStopTimeout());
+ }
}
- ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo;
-
- LOG.info("Recovering " + rbw);
-
- // Stop the previous writer
- rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout());
- rbw.setWriter(Thread.currentThread());
+ }
+ private synchronized ReplicaHandler recoverRbwImpl(ReplicaBeingWritten rbw,
+ ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
+ throws IOException {
// check generation stamp
long replicaGenerationStamp = rbw.getGenerationStamp();
if (replicaGenerationStamp < b.getGenerationStamp() ||
@@ -1419,7 +1458,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
long numBytes = rbw.getNumBytes();
if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd){
throw new ReplicaNotFoundException("Unmatched length replica " +
- replicaInfo + ": BytesAcked = " + bytesAcked +
+ rbw + ": BytesAcked = " + bytesAcked +
" BytesRcvd = " + numBytes + " are not in the range of [" +
minBytesRcvd + ", " + maxBytesRcvd + "].");
}
@@ -2351,8 +2390,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
@Override // FsDatasetSpi
- public synchronized ReplicaRecoveryInfo initReplicaRecovery(
- RecoveringBlock rBlock) throws IOException {
+ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
+ throws IOException {
return initReplicaRecovery(rBlock.getBlock().getBlockPoolId(), volumeMap,
rBlock.getBlock().getLocalBlock(), rBlock.getNewGenerationStamp(),
datanode.getDnConf().getXceiverStopTimeout());
@@ -2361,6 +2400,20 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
/** static version of {@link #initReplicaRecovery(RecoveringBlock)}. */
static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
Block block, long recoveryId, long xceiverStopTimeout) throws IOException {
+ while (true) {
+ try {
+ synchronized (map.getMutex()) {
+ return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
+ }
+ } catch (MustStopExistingWriter e) {
+ e.getReplica().stopWriter(xceiverStopTimeout);
+ }
+ }
+ }
+
+ static ReplicaRecoveryInfo initReplicaRecoveryImpl(String bpid, ReplicaMap map,
+ Block block, long recoveryId)
+ throws IOException, MustStopExistingWriter {
final ReplicaInfo replica = map.get(bpid, block.getBlockId());
LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
+ ", replica=" + replica);
@@ -2373,7 +2426,9 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
//stop writer if there is any
if (replica instanceof ReplicaInPipeline) {
final ReplicaInPipeline rip = (ReplicaInPipeline)replica;
- rip.stopWriter(xceiverStopTimeout);
+ if (!rip.attemptToSetWriter(null, Thread.currentThread())) {
+ throw new MustStopExistingWriter(rip);
+ }
//check replica bytes on disk.
if (rip.getBytesOnDisk() < rip.getVisibleLength()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b1a818/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
index 34c9f2e..192702e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
@@ -233,7 +233,7 @@ class ReplicaMap {
* Give access to mutex used for synchronizing ReplicasMap
* @return object used as lock
*/
- Object getMutext() {
+ Object getMutex() {
return mutex;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b1a818/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index 7e2bc0a..751089f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -42,10 +42,12 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.collect.Iterators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -92,7 +94,9 @@ import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -124,6 +128,9 @@ public class TestBlockRecovery {
private final static ExtendedBlock block = new ExtendedBlock(POOL_ID,
BLOCK_ID, BLOCK_LEN, GEN_STAMP);
+ @Rule
+ public TestName currentTestName = new TestName();
+
private static final int CELL_SIZE =
StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
private static final int bytesPerChecksum = 512;
@@ -153,6 +160,9 @@ public class TestBlockRecovery {
GenericTestUtils.setLogLevel(LOG, Level.ALL);
}
+ private final long
+ TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS = 1000000000L;
+
/**
* Starts an instance of DataNode
* @throws IOException
@@ -165,6 +175,12 @@ public class TestBlockRecovery {
conf.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
conf.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0");
+ if (currentTestName.getMethodName().equals(
+ "testInitReplicaRecoveryDoesNotHogLock")) {
+ // This test requires a very long value for the xceiver stop timeout.
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY,
+ TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS);
+ }
conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
FileSystem.setDefaultUri(conf,
"hdfs://" + NN_ADDR.getHostName() + ":" + NN_ADDR.getPort());
@@ -297,7 +313,7 @@ public class TestBlockRecovery {
* Two replicas are in Finalized state
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testFinalizedReplicas () throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -336,7 +352,7 @@ public class TestBlockRecovery {
* One replica is Finalized and another is RBW.
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testFinalizedRbwReplicas() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -378,7 +394,7 @@ public class TestBlockRecovery {
* One replica is Finalized and another is RWR.
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testFinalizedRwrReplicas() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -420,7 +436,7 @@ public class TestBlockRecovery {
* Two replicas are RBW.
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testRBWReplicas() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -444,7 +460,7 @@ public class TestBlockRecovery {
* One replica is RBW and another is RWR.
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testRBW_RWRReplicas() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -468,7 +484,7 @@ public class TestBlockRecovery {
* Two replicas are RWR.
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testRWRReplicas() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -504,7 +520,7 @@ public class TestBlockRecovery {
* @throws IOException
* in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testRecoveryInProgressException()
throws IOException, InterruptedException {
if(LOG.isDebugEnabled()) {
@@ -529,7 +545,7 @@ public class TestBlockRecovery {
* @throws IOException
* in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testErrorReplicas() throws IOException, InterruptedException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -556,7 +572,7 @@ public class TestBlockRecovery {
*
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testZeroLenReplicas() throws IOException, InterruptedException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -596,7 +612,7 @@ public class TestBlockRecovery {
*
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testFailedReplicaUpdate() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -618,7 +634,7 @@ public class TestBlockRecovery {
*
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testNoReplicaUnderRecovery() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -643,7 +659,7 @@ public class TestBlockRecovery {
*
* @throws IOException in case of an error
*/
- @Test
+ @Test(timeout=60000)
public void testNotMatchedReplicaID() throws IOException {
if(LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -744,7 +760,7 @@ public class TestBlockRecovery {
* throw an exception.
* @throws Exception
*/
- @Test
+ @Test(timeout=60000)
public void testRURReplicas() throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("Running " + GenericTestUtils.getMethodName());
@@ -775,7 +791,7 @@ public class TestBlockRecovery {
}
}
- @Test
+ @Test(timeout=60000)
public void testSafeLength() throws Exception {
// hard coded policy to work with hard coded test suite
ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager
@@ -799,4 +815,97 @@ public class TestBlockRecovery {
recoveryTask.getSafeLength(syncList));
}
}
+
+ /**
+ * Test that initReplicaRecovery does not hold the lock for an unreasonable
+ * amount of time if a writer is taking a long time to stop.
+ */
+ @Test(timeout=60000)
+ public void testInitReplicaRecoveryDoesNotHogLock() throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Running " + GenericTestUtils.getMethodName());
+ }
+ // We need a long value for the data xceiver stop timeout.
+ // Otherwise the timeout will trigger, and we will not have tested that
+ // thread join was done locklessly.
+ Assert.assertEquals(
+ TEST_LOCK_HOG_DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS,
+ dn.getDnConf().getXceiverStopTimeout());
+ final Semaphore progressParent = new Semaphore(0);
+ final Semaphore terminateSlowWorker = new Semaphore(0);
+ final AtomicBoolean failure = new AtomicBoolean(false);
+ Collection<RecoveringBlock> recoveringBlocks =
+ initRecoveringBlocks();
+ final RecoveringBlock recoveringBlock =
+ Iterators.get(recoveringBlocks.iterator(), 0);
+ final ExtendedBlock block = recoveringBlock.getBlock();
+ Thread slowWorker = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // Register this thread as the writer for the recoveringBlock.
+ LOG.debug("slowWorker creating rbw");
+ ReplicaHandler replicaHandler =
+ spyDN.data.createRbw(StorageType.DISK, block, false);
+ replicaHandler.close();
+ LOG.debug("slowWorker created rbw");
+ // Tell the parent thread to start progressing.
+ progressParent.release();
+ while (true) {
+ try {
+ terminateSlowWorker.acquire();
+ break;
+ } catch (InterruptedException e) {
+ // Ignore interrupted exceptions so that the waitingWorker thread
+ // will have to wait for us.
+ }
+ }
+ LOG.debug("slowWorker exiting");
+ } catch (Throwable t) {
+ LOG.error("slowWorker got exception", t);
+ failure.set(true);
+ }
+ }
+ });
+ // Start the slow worker thread and wait for it to take ownership of the
+ // ReplicaInPipeline
+ slowWorker.start();
+ while (true) {
+ try {
+ progressParent.acquire();
+ break;
+ } catch (InterruptedException e) {
+ // Ignore interrupted exceptions
+ }
+ }
+
+ // Start a worker thread which will wait for the slow worker thread.
+ Thread waitingWorker = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // Attempt to terminate the other worker thread and take ownership
+ // of the ReplicaInPipeline.
+ LOG.debug("waitingWorker initiating recovery");
+ spyDN.initReplicaRecovery(recoveringBlock);
+ LOG.debug("waitingWorker initiated recovery");
+ } catch (Throwable t) {
+ GenericTestUtils.assertExceptionContains("meta does not exist", t);
+ }
+ }
+ });
+ waitingWorker.start();
+
+ // Do an operation that requires the lock. This should not be blocked
+ // by the replica recovery in progress.
+ spyDN.getFSDataset().getReplicaString(
+ recoveringBlock.getBlock().getBlockPoolId(),
+ recoveringBlock.getBlock().getBlockId());
+
+ // Wait for the two worker threads to exit normally.
+ terminateSlowWorker.release();
+ slowWorker.join();
+ waitingWorker.join();
+ Assert.assertFalse("The slowWriter thread failed.", failure.get());
+ }
}
[43/50] [abbrv] hadoop git commit: YARN-4311. Removing nodes from
include and exclude lists will not remove them from decommissioned nodes
list. Contributed by Kuhu Shukla
Posted by ae...@apache.org.
YARN-4311. Removing nodes from include and exclude lists will not remove them from decommissioned nodes list. Contributed by Kuhu Shukla
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cbcd4a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cbcd4a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cbcd4a4
Branch: refs/heads/HDFS-1312
Commit: 1cbcd4a491e6a57d466c2897335614dc6770b475
Parents: 776b549
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Apr 5 13:40:19 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Apr 5 13:40:19 2016 +0000
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 9 +
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +
.../src/main/resources/yarn-default.xml | 13 ++
.../resourcemanager/NodesListManager.java | 104 ++++++++-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../resourcemanager/ResourceTrackerService.java | 8 +-
.../server/resourcemanager/rmnode/RMNode.java | 4 +
.../resourcemanager/rmnode/RMNodeImpl.java | 22 +-
.../yarn/server/resourcemanager/MockNodes.java | 9 +
.../TestResourceTrackerService.java | 216 +++++++++++++++++--
.../webapp/TestRMWebServicesNodes.java | 12 +-
12 files changed, 387 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 92d586b..951f5a8 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -199,6 +199,15 @@ public class NodeInfo {
public ResourceUtilization getNodeUtilization() {
return null;
}
+
+ @Override
+ public long getUntrackedTimeStamp() {
+ return 0;
+ }
+
+ @Override
+ public void setUntrackedTimeStamp(long timeStamp) {
+ }
}
public static RMNode newNodeInfo(String rackName, String hostName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 2e9cccb..e5013c4 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -188,4 +188,13 @@ public class RMNodeWrapper implements RMNode {
public ResourceUtilization getNodeUtilization() {
return node.getNodeUtilization();
}
+
+ @Override
+ public long getUntrackedTimeStamp() {
+ return 0;
+ }
+
+ @Override
+ public void setUntrackedTimeStamp(long timeStamp) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8acee57..66b293f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -648,6 +648,15 @@ public class YarnConfiguration extends Configuration {
"NONE";
/**
+ * Timeout(msec) for an untracked node to remain in shutdown or decommissioned
+ * state.
+ */
+ public static final String RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC =
+ RM_PREFIX + "node-removal-untracked.timeout-ms";
+ public static final int
+ DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC = 60000;
+
+ /**
* RM proxy users' prefix
*/
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 506cf3d..9e8b5e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2722,4 +2722,17 @@
<name>yarn.timeline-service.webapp.rest-csrf.methods-to-ignore</name>
<value>GET,OPTIONS,HEAD</value>
</property>
+
+ <property>
+ <description>
+ The least amount of time(msec.) an inactive (decommissioned or shutdown) node can
+ stay in the nodes list of the resourcemanager after being declared untracked.
+ A node is marked untracked if and only if it is absent from both include and
+ exclude nodemanager lists on the RM. All inactive nodes are checked twice per
+ timeout interval or every 10 minutes, whichever is lesser, and marked appropriately.
+ The same is done when refreshNodes command (graceful or otherwise) is invoked.
+ </description>
+ <name>yarn.resourcemanager.node-removal-untracked.timeout-ms</name>
+ <value>60000</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index ec2708e..65a9d94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.net.Node;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.HostsFileReader;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -68,6 +69,8 @@ public class NodesListManager extends CompositeService implements
private String excludesFile;
private Resolver resolver;
+ private Timer removalTimer;
+ private int nodeRemovalCheckInterval;
public NodesListManager(RMContext rmContext) {
super(NodesListManager.class.getName());
@@ -105,9 +108,56 @@ public class NodesListManager extends CompositeService implements
} catch (IOException ioe) {
disableHostsFileReader(ioe);
}
+
+ final int nodeRemovalTimeout =
+ conf.getInt(
+ YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC,
+ YarnConfiguration.
+ DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC);
+ nodeRemovalCheckInterval = (Math.min(nodeRemovalTimeout/2,
+ 600000));
+ removalTimer = new Timer("Node Removal Timer");
+
+ removalTimer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ long now = Time.monotonicNow();
+ for (Map.Entry<NodeId, RMNode> entry :
+ rmContext.getInactiveRMNodes().entrySet()) {
+ NodeId nodeId = entry.getKey();
+ RMNode rmNode = entry.getValue();
+ if (isUntrackedNode(rmNode.getHostName())) {
+ if (rmNode.getUntrackedTimeStamp() == 0) {
+ rmNode.setUntrackedTimeStamp(now);
+ } else if (now - rmNode.getUntrackedTimeStamp() >
+ nodeRemovalTimeout) {
+ RMNode result = rmContext.getInactiveRMNodes().remove(nodeId);
+ if (result != null) {
+ ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
+ if (rmNode.getState() == NodeState.SHUTDOWN) {
+ clusterMetrics.decrNumShutdownNMs();
+ } else {
+ clusterMetrics.decrDecommisionedNMs();
+ }
+ LOG.info("Removed "+result.getHostName() +
+ " from inactive nodes list");
+ }
+ }
+ } else {
+ rmNode.setUntrackedTimeStamp(0);
+ }
+ }
+ }
+ }, nodeRemovalCheckInterval, nodeRemovalCheckInterval);
+
super.serviceInit(conf);
}
+ @Override
+ public void serviceStop() {
+ removalTimer.cancel();
+ }
+
private void printConfiguredHosts() {
if (!LOG.isDebugEnabled()) {
return;
@@ -131,10 +181,13 @@ public class NodesListManager extends CompositeService implements
for (NodeId nodeId: rmContext.getRMNodes().keySet()) {
if (!isValidNode(nodeId.getHost())) {
+ RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ?
+ RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
+ new RMNodeEvent(nodeId, nodeEventType));
}
}
+ updateInactiveNodes();
}
private void refreshHostsReader(Configuration yarnConf) throws IOException,
@@ -172,6 +225,16 @@ public class NodesListManager extends CompositeService implements
}
@VisibleForTesting
+ public int getNodeRemovalCheckInterval() {
+ return nodeRemovalCheckInterval;
+ }
+
+ @VisibleForTesting
+ public void setNodeRemovalCheckInterval(int interval) {
+ this.nodeRemovalCheckInterval = interval;
+ }
+
+ @VisibleForTesting
public Resolver getResolver() {
return resolver;
}
@@ -374,6 +437,33 @@ public class NodesListManager extends CompositeService implements
return hostsReader;
}
+ private void updateInactiveNodes() {
+ long now = Time.monotonicNow();
+ for(Entry<NodeId, RMNode> entry :
+ rmContext.getInactiveRMNodes().entrySet()) {
+ NodeId nodeId = entry.getKey();
+ RMNode rmNode = entry.getValue();
+ if (isUntrackedNode(nodeId.getHost()) &&
+ rmNode.getUntrackedTimeStamp() == 0) {
+ rmNode.setUntrackedTimeStamp(now);
+ }
+ }
+ }
+
+ public boolean isUntrackedNode(String hostName) {
+ boolean untracked;
+ String ip = resolver.resolve(hostName);
+
+ synchronized (hostsReader) {
+ Set<String> hostsList = hostsReader.getHosts();
+ Set<String> excludeList = hostsReader.getExcludedHosts();
+ untracked = !hostsList.isEmpty() &&
+ !hostsList.contains(hostName) && !hostsList.contains(ip) &&
+ !excludeList.contains(hostName) && !excludeList.contains(ip);
+ }
+ return untracked;
+ }
+
/**
* Refresh the nodes gracefully
*
@@ -384,11 +474,13 @@ public class NodesListManager extends CompositeService implements
public void refreshNodesGracefully(Configuration conf) throws IOException,
YarnException {
refreshHostsReader(conf);
- for (Entry<NodeId, RMNode> entry:rmContext.getRMNodes().entrySet()) {
+ for (Entry<NodeId, RMNode> entry : rmContext.getRMNodes().entrySet()) {
NodeId nodeId = entry.getKey();
if (!isValidNode(nodeId.getHost())) {
+ RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ?
+ RMNodeEventType.SHUTDOWN : RMNodeEventType.GRACEFUL_DECOMMISSION;
this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(nodeId, RMNodeEventType.GRACEFUL_DECOMMISSION));
+ new RMNodeEvent(nodeId, nodeEventType));
} else {
// Recommissioning the nodes
if (entry.getValue().getState() == NodeState.DECOMMISSIONING) {
@@ -397,6 +489,7 @@ public class NodesListManager extends CompositeService implements
}
}
}
+ updateInactiveNodes();
}
/**
@@ -420,8 +513,11 @@ public class NodesListManager extends CompositeService implements
public void refreshNodesForcefully() {
for (Entry<NodeId, RMNode> entry : rmContext.getRMNodes().entrySet()) {
if (entry.getValue().getState() == NodeState.DECOMMISSIONING) {
+ RMNodeEventType nodeEventType =
+ isUntrackedNode(entry.getKey().getHost()) ?
+ RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(entry.getKey(), RMNodeEventType.DECOMMISSION));
+ new RMNodeEvent(entry.getKey(), nodeEventType));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index e19d55e..1318d58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -87,7 +87,7 @@ public class RMServerUtils {
acceptedStates.contains(NodeState.LOST) ||
acceptedStates.contains(NodeState.REBOOTED)) {
for (RMNode rmNode : context.getInactiveRMNodes().values()) {
- if (acceptedStates.contains(rmNode.getState())) {
+ if ((rmNode != null) && acceptedStates.contains(rmNode.getState())) {
results.add(rmNode);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index b0bc565..238e5bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -320,7 +320,8 @@ public class ResourceTrackerService extends AbstractService implements
}
// Check if this node is a 'valid' node
- if (!this.nodesListManager.isValidNode(host)) {
+ if (!this.nodesListManager.isValidNode(host) ||
+ this.nodesListManager.isUntrackedNode(host)) {
String message =
"Disallowed NodeManager from " + host
+ ", Sending SHUTDOWN signal to the NodeManager.";
@@ -451,8 +452,9 @@ public class ResourceTrackerService extends AbstractService implements
// 1. Check if it's a valid (i.e. not excluded) node, if not, see if it is
// in decommissioning.
- if (!this.nodesListManager.isValidNode(nodeId.getHost())
- && !isNodeInDecommissioning(nodeId)) {
+ if ((!this.nodesListManager.isValidNode(nodeId.getHost()) &&
+ !isNodeInDecommissioning(nodeId)) ||
+ this.nodesListManager.isUntrackedNode(nodeId.getHost())) {
String message =
"Disallowed NodeManager nodeId: " + nodeId + " hostname: "
+ nodeId.getHost();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index d8df9f1..e599576 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -168,4 +168,8 @@ public interface RMNode {
NodeHeartbeatResponse response);
public List<Container> pullNewlyIncreasedContainers();
+
+ long getUntrackedTimeStamp();
+
+ void setUntrackedTimeStamp(long timer);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 5f8317e..4260861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -120,6 +121,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private long lastHealthReportTime;
private String nodeManagerVersion;
+ private long timeStamp;
/* Aggregated resource utilization for the containers. */
private ResourceUtilization containersUtilization;
/* Resource utilization for the node. */
@@ -259,6 +261,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
.addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING,
RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition())
+ .addTransition(NodeState.DECOMMISSIONING, NodeState.SHUTDOWN,
+ RMNodeEventType.SHUTDOWN,
+ new DeactivateNodeTransition(NodeState.SHUTDOWN))
// TODO (in YARN-3223) update resource when container finished.
.addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING,
@@ -346,6 +351,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.healthReport = "Healthy";
this.lastHealthReportTime = System.currentTimeMillis();
this.nodeManagerVersion = nodeManagerVersion;
+ this.timeStamp = 0;
this.latestNodeHeartBeatResponse.setResponseId(0);
@@ -1011,7 +1017,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
/**
- * Put a node in deactivated (decommissioned) status.
+ * Put a node in deactivated (decommissioned or shutdown) status.
* @param rmNode
* @param finalState
*/
@@ -1028,6 +1034,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now "
+ finalState);
rmNode.context.getInactiveRMNodes().put(rmNode.nodeId, rmNode);
+ if (finalState == NodeState.SHUTDOWN &&
+ rmNode.context.getNodesListManager().isUntrackedNode(rmNode.hostName)) {
+ rmNode.setUntrackedTimeStamp(Time.monotonicNow());
+ }
}
/**
@@ -1383,4 +1393,14 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
public Resource getOriginalTotalCapability() {
return this.originalTotalCapability;
}
+
+ @Override
+ public long getUntrackedTimeStamp() {
+ return this.timeStamp;
+ }
+
+ @Override
+ public void setUntrackedTimeStamp(long ts) {
+ this.timeStamp = ts;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 89aff29..921b18e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -260,6 +260,15 @@ public class MockNodes {
public ResourceUtilization getNodeUtilization() {
return this.nodeUtilization;
}
+
+ @Override
+ public long getUntrackedTimeStamp() {
+ return 0;
+ }
+
+ @Override
+ public void setUntrackedTimeStamp(long timeStamp) {
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 9ed79a3..dd37b67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -31,6 +31,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
@@ -48,8 +50,6 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
@@ -141,12 +141,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.getNodesListManager().refreshNodes(conf);
- checkDecommissionedNMCount(rm, ++metricCount);
+ checkShutdownNMCount(rm, ++metricCount);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
Assert
- .assertEquals(1, ClusterMetrics.getMetrics().getNumDecommisionedNMs());
+ .assertEquals(1, ClusterMetrics.getMetrics().getNumShutdownNMs());
nodeHeartbeat = nm2.nodeHeartbeat(true);
Assert.assertTrue("Node is not decommisioned.", NodeAction.SHUTDOWN
@@ -155,7 +155,8 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nodeHeartbeat = nm3.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
Assert.assertEquals(metricCount, ClusterMetrics.getMetrics()
- .getNumDecommisionedNMs());
+ .getNumShutdownNMs());
+ rm.stop();
}
/**
@@ -228,7 +229,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
MockNM nm2 = rm.registerNode("host2:5678", 10240);
ClusterMetrics metrics = ClusterMetrics.getMetrics();
assert(metrics != null);
- int initialMetricCount = metrics.getNumDecommisionedNMs();
+ int initialMetricCount = metrics.getNumShutdownNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertEquals(
NodeAction.NORMAL,
@@ -241,16 +242,16 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
.getAbsolutePath());
rm.getNodesListManager().refreshNodes(conf);
- checkDecommissionedNMCount(rm, ++initialMetricCount);
+ checkShutdownNMCount(rm, ++initialMetricCount);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertEquals(
- "Node should not have been decomissioned.",
+ "Node should not have been shutdown.",
NodeAction.NORMAL,
nodeHeartbeat.getNodeAction());
- nodeHeartbeat = nm2.nodeHeartbeat(true);
- Assert.assertEquals("Node should have been decomissioned but is in state" +
- nodeHeartbeat.getNodeAction(),
- NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ NodeState nodeState =
+ rm.getRMContext().getInactiveRMNodes().get(nm2.getNodeId()).getState();
+ Assert.assertEquals("Node should have been shutdown but is in state" +
+ nodeState, NodeState.SHUTDOWN, nodeState);
}
/**
@@ -1123,8 +1124,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.start();
ResourceTrackerService resourceTrackerService = rm
.getResourceTrackerService();
- int shutdownNMsCount = ClusterMetrics.getMetrics()
- .getNumShutdownNMs();
int decommisionedNMsCount = ClusterMetrics.getMetrics()
.getNumDecommisionedNMs();
@@ -1149,10 +1148,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.getNodesListManager().refreshNodes(conf);
NodeHeartbeatResponse heartbeatResponse = nm1.nodeHeartbeat(true);
Assert.assertEquals(NodeAction.SHUTDOWN, heartbeatResponse.getNodeAction());
+ int shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
checkShutdownNMCount(rm, shutdownNMsCount);
- checkDecommissionedNMCount(rm, ++decommisionedNMsCount);
+ checkDecommissionedNMCount(rm, decommisionedNMsCount);
request.setNodeId(nm1.getNodeId());
resourceTrackerService.unRegisterNodeManager(request);
+ shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
checkShutdownNMCount(rm, shutdownNMsCount);
checkDecommissionedNMCount(rm, decommisionedNMsCount);
@@ -1168,8 +1169,9 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.getNodesListManager().refreshNodes(conf);
request.setNodeId(nm2.getNodeId());
resourceTrackerService.unRegisterNodeManager(request);
- checkShutdownNMCount(rm, shutdownNMsCount);
- checkDecommissionedNMCount(rm, ++decommisionedNMsCount);
+ checkShutdownNMCount(rm, ++shutdownNMsCount);
+ checkDecommissionedNMCount(rm, decommisionedNMsCount);
+ rm.stop();
}
@Test(timeout = 30000)
@@ -1304,6 +1306,186 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.stop();
}
+ /**
+ * Remove a node from all lists and check if its forgotten
+ */
+ @Test
+ public void testNodeRemovalNormally() throws Exception {
+ testNodeRemovalUtil(false);
+ }
+
+ @Test
+ public void testNodeRemovalGracefully() throws Exception {
+ testNodeRemovalUtil(true);
+ }
+
+ public void refreshNodesOption(boolean doGraceful, Configuration conf)
+ throws Exception {
+ if (doGraceful) {
+ rm.getNodesListManager().refreshNodesGracefully(conf);
+ } else {
+ rm.getNodesListManager().refreshNodes(conf);
+ }
+ }
+
+ public void testNodeRemovalUtil(boolean doGraceful) throws Exception {
+ Configuration conf = new Configuration();
+ int timeoutValue = 500;
+ File excludeHostFile = new File(TEMP_DIR + File.separator +
+ "excludeHostFile.txt");
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, "");
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, "");
+ conf.setInt(YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC,
+ timeoutValue);
+ CountDownLatch latch = new CountDownLatch(1);
+ rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+ RMContext rmContext = rm.getRMContext();
+ refreshNodesOption(doGraceful, conf);
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ MockNM nm3 = rm.registerNode("localhost:4433", 1024);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert (metrics != null);
+
+ //check all 3 nodes joined in as NORMAL
+ NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
+ nodeHeartbeat = nm3.nodeHeartbeat(true);
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
+ rm.drainEvents();
+ Assert.assertEquals("All 3 nodes should be active",
+ metrics.getNumActiveNMs(), 3);
+
+ //Remove nm2 from include list, should now be shutdown with timer test
+ String ip = NetUtils.normalizeHostName("localhost");
+ writeToHostsFile("host1", ip);
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ refreshNodesOption(doGraceful, conf);
+ nm1.nodeHeartbeat(true);
+ rm.drainEvents();
+ Assert.assertTrue("Node should not be in active node list",
+ !rmContext.getRMNodes().containsKey(nm2.getNodeId()));
+
+ RMNode rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertEquals("Node should be in inactive node list",
+ rmNode.getState(), NodeState.SHUTDOWN);
+ Assert.assertEquals("Active nodes should be 2",
+ metrics.getNumActiveNMs(), 2);
+ Assert.assertEquals("Shutdown nodes should be 1",
+ metrics.getNumShutdownNMs(), 1);
+
+ int nodeRemovalTimeout =
+ conf.getInt(
+ YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC,
+ YarnConfiguration.
+ DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC);
+ int nodeRemovalInterval =
+ rmContext.getNodesListManager().getNodeRemovalCheckInterval();
+ long maxThreadSleeptime = nodeRemovalInterval + nodeRemovalTimeout;
+ latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
+
+ rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertEquals("Node should have been forgotten!",
+ rmNode, null);
+ Assert.assertEquals("Shutdown nodes should be 0 now",
+ metrics.getNumShutdownNMs(), 0);
+
+ //Check node removal and re-addition before timer expires
+ writeToHostsFile("host1", ip, "host2");
+ refreshNodesOption(doGraceful, conf);
+ nm2 = rm.registerNode("host2:5678", 10240);
+ rm.drainEvents();
+ writeToHostsFile("host1", ip);
+ refreshNodesOption(doGraceful, conf);
+ rm.drainEvents();
+ rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertEquals("Node should be shutdown",
+ rmNode.getState(), NodeState.SHUTDOWN);
+ Assert.assertEquals("Active nodes should be 2",
+ metrics.getNumActiveNMs(), 2);
+ Assert.assertEquals("Shutdown nodes should be 1",
+ metrics.getNumShutdownNMs(), 1);
+
+ //add back the node before timer expires
+ latch.await(maxThreadSleeptime - 2000, TimeUnit.MILLISECONDS);
+ writeToHostsFile("host1", ip, "host2");
+ refreshNodesOption(doGraceful, conf);
+ nm2 = rm.registerNode("host2:5678", 10240);
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ rm.drainEvents();
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
+ Assert.assertEquals("Shutdown nodes should be 0 now",
+ metrics.getNumShutdownNMs(), 0);
+ Assert.assertEquals("All 3 nodes should be active",
+ metrics.getNumActiveNMs(), 3);
+
+ //Decommission this node, check timer doesn't remove it
+ writeToHostsFile("host1", "host2", ip);
+ writeToHostsFile(excludeHostFile, "host2");
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, excludeHostFile
+ .getAbsolutePath());
+ refreshNodesOption(doGraceful, conf);
+ rm.drainEvents();
+ rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertTrue("Node should be DECOMMISSIONED or DECOMMISSIONING",
+ (rmNode.getState() == NodeState.DECOMMISSIONED) ||
+ (rmNode.getState() == NodeState.DECOMMISSIONING));
+ if (rmNode.getState() == NodeState.DECOMMISSIONED) {
+ Assert.assertEquals("Decommissioned/ing nodes should be 1 now",
+ metrics.getNumDecommisionedNMs(), 1);
+ }
+ latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
+
+ rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertTrue("Node should be DECOMMISSIONED or DECOMMISSIONING",
+ (rmNode.getState() == NodeState.DECOMMISSIONED) ||
+ (rmNode.getState() == NodeState.DECOMMISSIONING));
+ if (rmNode.getState() == NodeState.DECOMMISSIONED) {
+ Assert.assertEquals("Decommissioned/ing nodes should be 1 now",
+ metrics.getNumDecommisionedNMs(), 1);
+ }
+
+ //Test decommed/ing node that transitions to untracked,timer should remove
+ writeToHostsFile("host1", ip, "host2");
+ writeToHostsFile(excludeHostFile, "host2");
+ refreshNodesOption(doGraceful, conf);
+ nm1.nodeHeartbeat(true);
+ //nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
+ rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertNotEquals("Timer for this node was not canceled!",
+ rmNode, null);
+ Assert.assertTrue("Node should be DECOMMISSIONED or DECOMMISSIONING",
+ (rmNode.getState() == NodeState.DECOMMISSIONED) ||
+ (rmNode.getState() == NodeState.DECOMMISSIONING));
+
+ writeToHostsFile("host1", ip);
+ writeToHostsFile(excludeHostFile, "");
+ refreshNodesOption(doGraceful, conf);
+ latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
+ rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId());
+ Assert.assertEquals("Node should have been forgotten!",
+ rmNode, null);
+ Assert.assertEquals("Shutdown nodes should be 0 now",
+ metrics.getNumDecommisionedNMs(), 0);
+ Assert.assertEquals("Shutdown nodes should be 0 now",
+ metrics.getNumShutdownNMs(), 0);
+ Assert.assertEquals("Active nodes should be 2",
+ metrics.getNumActiveNMs(), 2);
+
+ rm.stop();
+ }
+
private void writeToHostsFile(String... hosts) throws IOException {
writeToHostsFile(hostFile, hosts);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cbcd4a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 3fd1fd5..4b6ca12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -272,8 +272,10 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nodeId);
WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "",
info.getString("nodeHTTPAddress"));
- WebServicesTestUtils.checkStringMatch("state", rmNode.getState()
- .toString(), info.getString("state"));
+ if (rmNode != null) {
+ WebServicesTestUtils.checkStringMatch("state",
+ rmNode.getState().toString(), info.getString("state"));
+ }
}
}
@@ -304,8 +306,10 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(nodeId);
WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "",
info.getString("nodeHTTPAddress"));
- WebServicesTestUtils.checkStringMatch("state",
- rmNode.getState().toString(), info.getString("state"));
+ if (rmNode != null) {
+ WebServicesTestUtils.checkStringMatch("state",
+ rmNode.getState().toString(), info.getString("state"));
+ }
}
@Test
[31/50] [abbrv] hadoop git commit: HADOOP-12967. Remove
FileUtil#copyMerge. Contributed by Brahma Reddy Battula.
Posted by ae...@apache.org.
HADOOP-12967. Remove FileUtil#copyMerge. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da614ca5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da614ca5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da614ca5
Branch: refs/heads/HDFS-1312
Commit: da614ca5dc26562d7ecd5d7c5743fa52c3c17342
Parents: 1e6f929
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Apr 4 17:46:56 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Apr 4 17:48:08 2016 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 42 --------------
.../java/org/apache/hadoop/fs/TestFileUtil.java | 58 --------------------
2 files changed, 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da614ca5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index b855c48..e2d6ecd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -23,7 +23,6 @@ import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
@@ -381,47 +380,6 @@ public class FileUtil {
}
- @Deprecated
- /** Copy all files in a directory to one output file (merge). */
- public static boolean copyMerge(FileSystem srcFS, Path srcDir,
- FileSystem dstFS, Path dstFile,
- boolean deleteSource,
- Configuration conf, String addString) throws IOException {
- dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
-
- if (!srcFS.getFileStatus(srcDir).isDirectory())
- return false;
-
- OutputStream out = dstFS.create(dstFile);
-
- try {
- FileStatus contents[] = srcFS.listStatus(srcDir);
- Arrays.sort(contents);
- for (int i = 0; i < contents.length; i++) {
- if (contents[i].isFile()) {
- InputStream in = srcFS.open(contents[i].getPath());
- try {
- IOUtils.copyBytes(in, out, conf, false);
- if (addString!=null)
- out.write(addString.getBytes("UTF-8"));
-
- } finally {
- in.close();
- }
- }
- }
- } finally {
- out.close();
- }
-
-
- if (deleteSource) {
- return srcFS.delete(srcDir, true);
- } else {
- return true;
- }
- }
-
/** Copy local files to a FileSystem. */
public static boolean copy(File src,
FileSystem dstFS, Path dst,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da614ca5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index f7464b7..a9ef5c0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -19,11 +19,9 @@ package org.apache.hadoop.fs;
import org.junit.*;
-import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetAddress;
@@ -49,7 +47,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarOutputStream;
-import javax.print.attribute.URISyntax;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
@@ -526,61 +523,6 @@ public class TestFileUtil {
validateAndSetWritablePermissions(false, ret);
}
- @Test (timeout = 30000)
- public void testCopyMergeSingleDirectory() throws IOException {
- setupDirs();
- boolean copyMergeResult = copyMerge("partitioned", "tmp/merged");
- Assert.assertTrue("Expected successful copyMerge result.", copyMergeResult);
- File merged = new File(TEST_DIR, "tmp/merged");
- Assert.assertTrue("File tmp/merged must exist after copyMerge.",
- merged.exists());
- BufferedReader rdr = new BufferedReader(new FileReader(merged));
-
- try {
- Assert.assertEquals("Line 1 of merged file must contain \"foo\".",
- "foo", rdr.readLine());
- Assert.assertEquals("Line 2 of merged file must contain \"bar\".",
- "bar", rdr.readLine());
- Assert.assertNull("Expected end of file reading merged file.",
- rdr.readLine());
- }
- finally {
- rdr.close();
- }
- }
-
- /**
- * Calls FileUtil.copyMerge using the specified source and destination paths.
- * Both source and destination are assumed to be on the local file system.
- * The call will not delete source on completion and will not add an
- * additional string between files.
- * @param src String non-null source path.
- * @param dst String non-null destination path.
- * @return boolean true if the call to FileUtil.copyMerge was successful.
- * @throws IOException if an I/O error occurs.
- */
- @SuppressWarnings("deprecation")
- private boolean copyMerge(String src, String dst)
- throws IOException {
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.getLocal(conf);
- final boolean result;
-
- try {
- Path srcPath = new Path(TEST_ROOT_DIR, src);
- Path dstPath = new Path(TEST_ROOT_DIR, dst);
- boolean deleteSource = false;
- String addString = null;
- result = FileUtil.copyMerge(fs, srcPath, fs, dstPath, deleteSource, conf,
- addString);
- }
- finally {
- fs.close();
- }
-
- return result;
- }
-
/**
* Test that getDU is able to handle cycles caused due to symbolic links
* and that directory sizes are not added to the final calculated size
[32/50] [abbrv] hadoop git commit: YARN-4746. yarn web services
should convert parse failures of appId,
appAttemptId and containerId to 400. Contributed by Bibin A Chundatt
Posted by ae...@apache.org.
YARN-4746. yarn web services should convert parse failures of appId, appAttemptId and containerId to 400. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5092c941
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5092c941
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5092c941
Branch: refs/heads/HDFS-1312
Commit: 5092c94195a63bd2c3e36d5a74b4c061cea1b847
Parents: da614ca
Author: naganarasimha <na...@apache.com>
Authored: Mon Apr 4 16:25:03 2016 +0530
Committer: naganarasimha <na...@apache.com>
Committed: Mon Apr 4 16:25:03 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/util/ConverterUtils.java | 16 ++++++++--
.../hadoop/yarn/webapp/util/WebAppUtils.java | 22 ++++++++++++++
.../hadoop/yarn/server/webapp/WebServices.java | 22 +++++++++++---
.../nodemanager/webapp/NMWebServices.java | 6 ++--
.../webapp/TestNMWebServicesApps.java | 9 ++++--
.../resourcemanager/webapp/RMWebServices.java | 32 ++------------------
.../webapp/TestRMWebServicesApps.java | 24 +++++++++------
.../TestRMWebServicesAppsModification.java | 10 ++++--
8 files changed, 87 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
index e9674cf..acd29fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
@@ -122,8 +122,20 @@ public class ConverterUtils {
public static ApplicationId toApplicationId(RecordFactory recordFactory,
String appIdStr) {
Iterator<String> it = _split(appIdStr).iterator();
- it.next(); // prefix. TODO: Validate application prefix
- return toApplicationId(recordFactory, it);
+ if (!it.next().equals(APPLICATION_PREFIX)) {
+ throw new IllegalArgumentException("Invalid ApplicationId prefix: "
+ + appIdStr + ". The valid ApplicationId should start with prefix "
+ + APPLICATION_PREFIX);
+ }
+ try {
+ return toApplicationId(recordFactory, it);
+ } catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr,
+ n);
+ } catch (NoSuchElementException e) {
+ throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr,
+ e);
+ }
}
private static ApplicationId toApplicationId(RecordFactory recordFactory,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index f8e67ee..faf4a77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -33,9 +33,14 @@ import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.RMHAUtils;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
@Private
@Evolving
@@ -378,4 +383,21 @@ public class WebAppUtils {
}
return password;
}
+
+ public static ApplicationId parseApplicationId(RecordFactory recordFactory,
+ String appId) {
+ if (appId == null || appId.isEmpty()) {
+ throw new NotFoundException("appId, " + appId + ", is empty or null");
+ }
+ ApplicationId aid = null;
+ try {
+ aid = ConverterUtils.toApplicationId(recordFactory, appId);
+ } catch (Exception e) {
+ throw new BadRequestException(e);
+ }
+ if (aid == null) {
+ throw new NotFoundException("app with id " + appId + " not found");
+ }
+ return aid;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 40e40c9..19ea301 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -429,7 +429,12 @@ public class WebServices {
if (appId == null || appId.isEmpty()) {
throw new NotFoundException("appId, " + appId + ", is empty or null");
}
- ApplicationId aid = ConverterUtils.toApplicationId(appId);
+ ApplicationId aid = null;
+ try {
+ aid = ConverterUtils.toApplicationId(appId);
+ } catch (Exception e) {
+ throw new BadRequestException(e);
+ }
if (aid == null) {
throw new NotFoundException("appId is null");
}
@@ -442,8 +447,12 @@ public class WebServices {
throw new NotFoundException("appAttemptId, " + appAttemptId
+ ", is empty or null");
}
- ApplicationAttemptId aaid =
- ConverterUtils.toApplicationAttemptId(appAttemptId);
+ ApplicationAttemptId aaid = null;
+ try {
+ aaid = ConverterUtils.toApplicationAttemptId(appAttemptId);
+ } catch (Exception e) {
+ throw new BadRequestException(e);
+ }
if (aaid == null) {
throw new NotFoundException("appAttemptId is null");
}
@@ -455,7 +464,12 @@ public class WebServices {
throw new NotFoundException("containerId, " + containerId
+ ", is empty or null");
}
- ContainerId cid = ConverterUtils.toContainerId(containerId);
+ ContainerId cid = null;
+ try {
+ cid = ConverterUtils.toContainerId(containerId);
+ } catch (Exception e) {
+ throw new BadRequestException(e);
+ }
if (cid == null) {
throw new NotFoundException("containerId is null");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index de6d219..fddeb04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -143,10 +144,7 @@ public class NMWebServices {
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppInfo getNodeApp(@PathParam("appid") String appId) {
init();
- ApplicationId id = ConverterUtils.toApplicationId(recordFactory, appId);
- if (id == null) {
- throw new NotFoundException("app with id " + appId + " not found");
- }
+ ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
Application app = this.nmContext.getApplications().get(id);
if (app == null) {
throw new NotFoundException("app with id " + appId + " not found");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
index e274abb..3cd8faf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
@@ -565,11 +565,14 @@ public class TestNMWebServicesApps extends JerseyTestBase {
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringMatch("exception message",
- "For input string: \"foo\"", message);
+ "java.lang.IllegalArgumentException: Invalid ApplicationId prefix: "
+ + "app_foo_0000. The valid ApplicationId should start with prefix"
+ + " application",
+ message);
WebServicesTestUtils.checkStringMatch("exception type",
- "NumberFormatException", type);
+ "BadRequestException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
- "java.lang.NumberFormatException", classname);
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 0ed8a4e..8036af4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -662,14 +662,7 @@ public class RMWebServices extends WebServices {
public AppInfo getApp(@Context HttpServletRequest hsr,
@PathParam("appid") String appId) {
init();
- if (appId == null || appId.isEmpty()) {
- throw new NotFoundException("appId, " + appId + ", is empty or null");
- }
- ApplicationId id;
- id = ConverterUtils.toApplicationId(recordFactory, appId);
- if (id == null) {
- throw new NotFoundException("appId is null");
- }
+ ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
RMApp app = rm.getRMContext().getRMApps().get(id);
if (app == null) {
throw new NotFoundException("app with id: " + appId + " not found");
@@ -684,14 +677,7 @@ public class RMWebServices extends WebServices {
@PathParam("appid") String appId) {
init();
- if (appId == null || appId.isEmpty()) {
- throw new NotFoundException("appId, " + appId + ", is empty or null");
- }
- ApplicationId id;
- id = ConverterUtils.toApplicationId(recordFactory, appId);
- if (id == null) {
- throw new NotFoundException("appId is null");
- }
+ ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
RMApp app = rm.getRMContext().getRMApps().get(id);
if (app == null) {
throw new NotFoundException("app with id: " + appId + " not found");
@@ -1303,19 +1289,7 @@ public class RMWebServices extends WebServices {
}
private RMApp getRMAppForAppId(String appId) {
-
- if (appId == null || appId.isEmpty()) {
- throw new NotFoundException("appId, " + appId + ", is empty or null");
- }
- ApplicationId id;
- try {
- id = ConverterUtils.toApplicationId(recordFactory, appId);
- } catch (NumberFormatException e) {
- throw new NotFoundException("appId is invalid");
- }
- if (id == null) {
- throw new NotFoundException("appId is invalid");
- }
+ ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
RMApp app = rm.getRMContext().getRMApps().get(id);
if (app == null) {
throw new NotFoundException("app with id: " + appId + " not found");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 0328a7a..72cccf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1194,11 +1194,13 @@ public class TestRMWebServicesApps extends JerseyTestBase {
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringMatch("exception message",
- "For input string: \"invalid\"", message);
+ "java.lang.IllegalArgumentException: Invalid ApplicationId:"
+ + " application_invalid_12",
+ message);
WebServicesTestUtils.checkStringMatch("exception type",
- "NumberFormatException", type);
+ "BadRequestException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
- "java.lang.NumberFormatException", classname);
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
} finally {
rm.stop();
@@ -1500,15 +1502,17 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void testInvalidAppAttempts() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(CONTAINER_MB);
+ RMApp app = rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
try {
r.path("ws").path("v1").path("cluster").path("apps")
- .path("application_invalid_12").accept(MediaType.APPLICATION_JSON)
+ .path(app.getApplicationId().toString()).path("appattempts")
+ .path("appattempt_invalid_12_000001")
+ .accept(MediaType.APPLICATION_JSON)
.get(JSONObject.class);
- fail("should have thrown exception on invalid appid");
+ fail("should have thrown exception on invalid appAttempt");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
@@ -1521,11 +1525,13 @@ public class TestRMWebServicesApps extends JerseyTestBase {
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringMatch("exception message",
- "For input string: \"invalid\"", message);
+ "java.lang.IllegalArgumentException: Invalid AppAttemptId:"
+ + " appattempt_invalid_12_000001",
+ message);
WebServicesTestUtils.checkStringMatch("exception type",
- "NumberFormatException", type);
+ "BadRequestException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
- "java.lang.NumberFormatException", classname);
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
} finally {
rm.stop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5092c941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 9abcf9c..38b32e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -568,17 +568,21 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
amNodeManager.nodeHeartbeat(true);
String[] testAppIds = { "application_1391705042196_0001", "random_string" };
- for (String testAppId : testAppIds) {
+ for (int i = 0; i < testAppIds.length; i++) {
AppState info = new AppState("KILLED");
ClientResponse response =
- this.constructWebResource("apps", testAppId, "state")
+ this.constructWebResource("apps", testAppIds[i], "state")
.accept(MediaType.APPLICATION_XML)
.entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class);
if (!isAuthenticationEnabled()) {
assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
continue;
}
- assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ if (i == 0) {
+ assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
+ } else {
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ }
}
rm.stop();
}
[27/50] [abbrv] hadoop git commit: YARN-4657. Javadoc comment is
broken for Resources.multiplyByAndAddTo(). (Daniel Templeton via kasha)
Posted by ae...@apache.org.
YARN-4657. Javadoc comment is broken for Resources.multiplyByAndAddTo(). (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81d04cae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81d04cae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81d04cae
Branch: refs/heads/HDFS-1312
Commit: 81d04cae41182808ace5d86cdac7e4d71871eb1e
Parents: 5686caa
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Fri Apr 1 16:19:54 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Fri Apr 1 16:20:00 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/util/resource/Resources.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81d04cae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index b05d021..558f96c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -152,7 +152,7 @@ public class Resources {
}
/**
- * Multiply @param rhs by @param by, and add the result to @param lhs
+ * Multiply {@code rhs} by {@code by}, and add the result to {@code lhs}
* without creating any new {@link Resource} object
*/
public static Resource multiplyAndAddTo(
[09/50] [abbrv] hadoop git commit: HADOOP-12902. JavaDocs for
SignerSecretProvider are out-of-date in AuthenticationFilter. Contributed by
Gabor Liptak.
Posted by ae...@apache.org.
HADOOP-12902. JavaDocs for SignerSecretProvider are out-of-date in AuthenticationFilter. Contributed by Gabor Liptak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acca149e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acca149e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acca149e
Branch: refs/heads/HDFS-1312
Commit: acca149ec96f2932bebc492452a63a159de9ce47
Parents: 32c0c3e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Mar 31 16:04:47 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Mar 31 16:04:47 2016 +0900
----------------------------------------------------------------------
.../server/AuthenticationFilter.java | 33 ++++++++++----------
1 file changed, 16 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acca149e/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 4bdc808..5c93fd3 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -61,9 +61,9 @@ import java.util.*;
* <li>[#PREFIX#.]type: simple|kerberos|#CLASS#, 'simple' is short for the
* {@link PseudoAuthenticationHandler}, 'kerberos' is short for {@link KerberosAuthenticationHandler}, otherwise
* the full class name of the {@link AuthenticationHandler} must be specified.</li>
- * <li>[#PREFIX#.]signature.secret: when signer.secret.provider is set to
- * "string" or not specified, this is the value for the secret used to sign the
- * HTTP cookie.</li>
+ * <li>[#PREFIX#.]signature.secret.file: when signer.secret.provider is set to
+ * "file" or not specified, this is the location of file including the secret
+ * used to sign the HTTP cookie.</li>
* <li>[#PREFIX#.]token.validity: time -in seconds- that the generated token is
* valid before a new authentication is triggered, default value is
* <code>3600</code> seconds. This is also used for the rollover interval for
@@ -79,17 +79,16 @@ import java.util.*;
* </p>
* <p>
* Out of the box it provides 3 signer secret provider implementations:
- * "string", "random", and "zookeeper"
+ * "file", "random" and "zookeeper"
* </p>
* Additional signer secret providers are supported via the
* {@link SignerSecretProvider} class.
* <p>
* For the HTTP cookies mentioned above, the SignerSecretProvider is used to
* determine the secret to use for signing the cookies. Different
- * implementations can have different behaviors. The "string" implementation
- * simply uses the string set in the [#PREFIX#.]signature.secret property
- * mentioned above. The "random" implementation uses a randomly generated
- * secret that rolls over at the interval specified by the
+ * implementations can have different behaviors. The "file" implementation
+ * loads the secret from a specified file. The "random" implementation uses a
+ * randomly generated secret that rolls over at the interval specified by the
* [#PREFIX#.]token.validity mentioned above. The "zookeeper" implementation
* is like the "random" one, except that it synchronizes the random secret
* and rollovers between multiple servers; it's meant for HA services.
@@ -97,12 +96,12 @@ import java.util.*;
* The relevant configuration properties are:
* <ul>
* <li>signer.secret.provider: indicates the name of the SignerSecretProvider
- * class to use. Possible values are: "string", "random", "zookeeper", or a
- * classname. If not specified, the "string" implementation will be used with
- * [#PREFIX#.]signature.secret; and if that's not specified, the "random"
+ * class to use. Possible values are: "file", "random", "zookeeper", or a
+ * classname. If not specified, the "file" implementation will be used with
+ * [#PREFIX#.]signature.secret.file; and if that's not specified, the "random"
* implementation will be used.</li>
- * <li>[#PREFIX#.]signature.secret: When the "string" implementation is
- * specified, this value is used as the secret.</li>
+ * <li>[#PREFIX#.]signature.secret.file: When the "file" implementation is
+ * specified, this content of this file is used as the secret.</li>
* <li>[#PREFIX#.]token.validity: When the "random" or "zookeeper"
* implementations are specified, this value is used as the rollover
* interval.</li>
@@ -176,10 +175,10 @@ public class AuthenticationFilter implements Filter {
/**
* Constant for the configuration property that indicates the name of the
* SignerSecretProvider class to use.
- * Possible values are: "string", "random", "zookeeper", or a classname.
- * If not specified, the "string" implementation will be used with
- * SIGNATURE_SECRET; and if that's not specified, the "random" implementation
- * will be used.
+ * Possible values are: "file", "random", "zookeeper", or a classname.
+ * If not specified, the "file" implementation will be used with
+ * SIGNATURE_SECRET_FILE; and if that's not specified, the "random"
+ * implementation will be used.
*/
public static final String SIGNER_SECRET_PROVIDER =
"signer.secret.provider";
[26/50] [abbrv] hadoop git commit: Missing file for YARN-4895.
Posted by ae...@apache.org.
Missing file for YARN-4895.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5686caa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5686caa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5686caa9
Branch: refs/heads/HDFS-1312
Commit: 5686caa9fcb59759c9286385575f31e407a97c16
Parents: 82621e3
Author: Arun Suresh <as...@apache.org>
Authored: Fri Apr 1 15:58:13 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Apr 1 15:58:13 2016 -0700
----------------------------------------------------------------------
.../api/records/TestResourceUtilization.java | 63 ++++++++++++++++++++
1 file changed, 63 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5686caa9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java
new file mode 100644
index 0000000..5934846
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestResourceUtilization {
+
+ @Test
+ public void testResourceUtilization() {
+ ResourceUtilization u1 = ResourceUtilization.newInstance(10, 20, 0.5f);
+ ResourceUtilization u2 = ResourceUtilization.newInstance(u1);
+ ResourceUtilization u3 = ResourceUtilization.newInstance(10, 20, 0.5f);
+ ResourceUtilization u4 = ResourceUtilization.newInstance(20, 20, 0.5f);
+ ResourceUtilization u5 = ResourceUtilization.newInstance(30, 40, 0.8f);
+
+ Assert.assertEquals(u1, u2);
+ Assert.assertEquals(u1, u3);
+ Assert.assertNotEquals(u1, u4);
+ Assert.assertNotEquals(u2, u5);
+ Assert.assertNotEquals(u4, u5);
+
+ Assert.assertTrue(u1.hashCode() == u2.hashCode());
+ Assert.assertTrue(u1.hashCode() == u3.hashCode());
+ Assert.assertFalse(u1.hashCode() == u4.hashCode());
+ Assert.assertFalse(u2.hashCode() == u5.hashCode());
+ Assert.assertFalse(u4.hashCode() == u5.hashCode());
+
+ Assert.assertTrue(u1.getPhysicalMemory() == 10);
+ Assert.assertFalse(u1.getVirtualMemory() == 10);
+ Assert.assertTrue(u1.getCPU() == 0.5f);
+
+ Assert.assertEquals("<pmem:10, vmem:" + u1.getVirtualMemory()
+ + ", vCores:0.5>", u1.toString());
+
+ u1.addTo(10, 0, 0.0f);
+ Assert.assertNotEquals(u1, u2);
+ Assert.assertEquals(u1, u4);
+ u1.addTo(10, 20, 0.3f);
+ Assert.assertEquals(u1, u5);
+ u1.subtractFrom(10, 20, 0.3f);
+ Assert.assertEquals(u1, u4);
+ u1.subtractFrom(10, 0, 0.0f);
+ Assert.assertEquals(u1, u3);
+ }
+}
[34/50] [abbrv] hadoop git commit: HADOOP-11212.
NetUtils.wrapException to handle SocketException explicitly. (Contributed by
Steve Loughran)
Posted by ae...@apache.org.
HADOOP-11212. NetUtils.wrapException to handle SocketException explicitly. (Contributed by Steve Loughran)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7280550a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7280550a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7280550a
Branch: refs/heads/HDFS-1312
Commit: 7280550a8f668df8aa32e4630db4ead49e9b8b6d
Parents: 89c9347
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Apr 4 10:50:11 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Apr 4 10:50:11 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/net/NetUtils.java | 15 +++++--
.../org/apache/hadoop/net/TestNetUtils.java | 47 +++++++++++---------
2 files changed, 38 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7280550a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index 2c3661a..4050107 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -782,12 +782,21 @@ public class NetUtils {
+ ": " + exception
+ ";"
+ see("EOFException"));
+ } else if (exception instanceof SocketException) {
+ // Many of the predecessor exceptions are subclasses of SocketException,
+ // so must be handled before this
+ return wrapWithMessage(exception,
+ "Call From "
+ + localHost + " to " + destHost + ":" + destPort
+ + " failed on socket exception: " + exception
+ + ";"
+ + see("SocketException"));
}
else {
return (IOException) new IOException("Failed on local exception: "
- + exception
- + "; Host Details : "
- + getHostDetailsAsString(destHost, destPort, localHost))
+ + exception
+ + "; Host Details : "
+ + getHostDetailsAsString(destHost, destPort, localHost))
.initCause(exception);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7280550a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
index c93ede8..e59ac77 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
@@ -72,7 +72,7 @@ public class TestNetUtils {
* This is a regression test for HADOOP-6722.
*/
@Test
- public void testAvoidLoopbackTcpSockets() throws Exception {
+ public void testAvoidLoopbackTcpSockets() throws Throwable {
Configuration conf = new Configuration();
Socket socket = NetUtils.getDefaultSocketFactory(conf)
@@ -88,11 +88,11 @@ public class TestNetUtils {
fail("Should not have connected");
} catch (ConnectException ce) {
System.err.println("Got exception: " + ce);
- assertTrue(ce.getMessage().contains("resulted in a loopback"));
+ assertInException(ce, "resulted in a loopback");
} catch (SocketException se) {
// Some TCP stacks will actually throw their own Invalid argument exception
// here. This is also OK.
- assertTrue(se.getMessage().contains("Invalid argument"));
+ assertInException(se, "Invalid argument");
}
}
@@ -188,15 +188,11 @@ public class TestNetUtils {
}
@Test
- public void testVerifyHostnamesNoException() {
+ public void testVerifyHostnamesNoException() throws UnknownHostException {
String[] names = {"valid.host.com", "1.com"};
- try {
- NetUtils.verifyHostnames(names);
- } catch (UnknownHostException e) {
- fail("NetUtils.verifyHostnames threw unexpected UnknownHostException");
- }
+ NetUtils.verifyHostnames(names);
}
-
+
/**
* Test for {@link NetUtils#isLocalAddress(java.net.InetAddress)}
*/
@@ -267,7 +263,18 @@ public class TestNetUtils {
assertRemoteDetailsIncluded(wrapped);
assertInException(wrapped, "/EOFException");
}
-
+
+ @Test
+ public void testWrapSocketException() throws Throwable {
+ IOException wrapped = verifyExceptionClass(new SocketException("failed"),
+ SocketException.class);
+ assertInException(wrapped, "failed");
+ assertWikified(wrapped);
+ assertInException(wrapped, "localhost");
+ assertRemoteDetailsIncluded(wrapped);
+ assertInException(wrapped, "/SocketException");
+ }
+
@Test
public void testGetConnectAddress() throws IOException {
NetUtils.addStaticResolution("host", "127.0.0.1");
@@ -322,8 +329,8 @@ public class TestNetUtils {
String message = extractExceptionMessage(e);
if (!(message.contains(text))) {
throw new AssertionFailedError("Wrong text in message "
- + "\"" + message + "\""
- + " expected \"" + text + "\"")
+ + "\"" + message + "\""
+ + " expected \"" + text + "\"")
.initCause(e);
}
}
@@ -343,8 +350,8 @@ public class TestNetUtils {
String message = extractExceptionMessage(e);
if (message.contains(text)) {
throw new AssertionFailedError("Wrong text in message "
- + "\"" + message + "\""
- + " did not expect \"" + text + "\"")
+ + "\"" + message + "\""
+ + " did not expect \"" + text + "\"")
.initCause(e);
}
}
@@ -353,15 +360,13 @@ public class TestNetUtils {
Class expectedClass)
throws Throwable {
assertNotNull("Null Exception", e);
- IOException wrapped =
- NetUtils.wrapException("desthost", DEST_PORT,
- "localhost", LOCAL_PORT,
- e);
+ IOException wrapped = NetUtils.wrapException("desthost", DEST_PORT,
+ "localhost", LOCAL_PORT, e);
LOG.info(wrapped.toString(), wrapped);
if(!(wrapped.getClass().equals(expectedClass))) {
throw new AssertionFailedError("Wrong exception class; expected "
- + expectedClass
- + " got " + wrapped.getClass() + ": " + wrapped).initCause(wrapped);
+ + expectedClass
+ + " got " + wrapped.getClass() + ": " + wrapped).initCause(wrapped);
}
return wrapped;
}
[45/50] [abbrv] hadoop git commit: HDFS-10239. Fsshell mv fails if
port usage doesn't match in src and destination paths. Contributed by Kuhu
Shukla.
Posted by ae...@apache.org.
HDFS-10239. Fsshell mv fails if port usage doesn't match in src and destination paths. Contributed by Kuhu Shukla.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91746450
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91746450
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91746450
Branch: refs/heads/HDFS-1312
Commit: 917464505c0e930ebeb4c775d829e51c56a48686
Parents: 6be28bc
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Apr 5 09:07:24 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Apr 5 09:07:24 2016 -0500
----------------------------------------------------------------------
.../apache/hadoop/fs/shell/MoveCommands.java | 6 +++-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 31 ++++++++++++++++++++
2 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91746450/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
index 02a3b25..d359282 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
@@ -100,7 +100,11 @@ class MoveCommands {
@Override
protected void processPath(PathData src, PathData target) throws IOException {
- if (!src.fs.getUri().equals(target.fs.getUri())) {
+ String srcUri = src.fs.getUri().getScheme() + "://" +
+ src.fs.getUri().getHost();
+ String dstUri = target.fs.getUri().getScheme() + "://" +
+ target.fs.getUri().getHost();
+ if (!srcUri.equals(dstUri)) {
throw new PathIOException(src.toString(),
"Does not match target filesystem");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91746450/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 41cd5c0..b75ac11 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -559,6 +559,37 @@ public class TestDFSShell {
}
}
+ @Test
+ public void testMoveWithTargetPortEmpty() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .format(true)
+ .numDataNodes(2)
+ .nameNodePort(8020)
+ .waitSafeMode(true)
+ .build();
+ FileSystem srcFs = cluster.getFileSystem();
+ FsShell shell = new FsShell();
+ shell.setConf(conf);
+ String[] argv = new String[2];
+ argv[0] = "-mkdir";
+ argv[1] = "/testfile";
+ ToolRunner.run(shell, argv);
+ argv = new String[3];
+ argv[0] = "-mv";
+ argv[1] = srcFs.getUri() + "/testfile";
+ argv[2] = "hdfs://localhost/testfile2";
+ int ret = ToolRunner.run(shell, argv);
+ assertEquals("mv should have succeeded", 0, ret);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
@Test (timeout = 30000)
public void testURIPaths() throws Exception {
Configuration srcConf = new HdfsConfiguration();
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-1312
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48a8c9c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48a8c9c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48a8c9c3
Branch: refs/heads/HDFS-1312
Commit: 48a8c9c3e2cc3b24913a196dcf6b47ba93a07a92
Parents: 12b4cf7 500e5a5
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Apr 5 12:28:48 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Apr 5 12:28:48 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 6 +-
dev-support/bin/dist-tools-hooks-maker | 182 +++
dev-support/docker/Dockerfile | 8 +
.../server/AuthenticationFilter.java | 33 +-
.../JWTRedirectAuthenticationHandler.java | 16 +-
.../TestJWTRedirectAuthentictionHandler.java | 37 +-
.../hadoop-common/src/main/bin/hadoop | 22 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 6 +-
.../hadoop-common/src/main/bin/hadoop-daemon.sh | 6 +-
.../src/main/bin/hadoop-daemons.sh | 6 +-
.../src/main/bin/hadoop-functions.sh | 127 ++-
.../src/main/bin/hadoop-layout.sh.example | 22 +-
.../hadoop-common/src/main/bin/slaves.sh | 6 +-
.../hadoop-common/src/main/bin/start-all.sh | 4 +-
.../hadoop-common/src/main/bin/stop-all.sh | 4 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 31 +-
.../org/apache/hadoop/conf/Configuration.java | 13 +
.../crypto/key/kms/KMSClientProvider.java | 8 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 16 +-
.../hadoop/fs/CommonConfigurationKeys.java | 14 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 126 ++-
.../java/org/apache/hadoop/fs/FileUtil.java | 41 -
.../org/apache/hadoop/fs/PathIOException.java | 9 +
.../hadoop/fs/shell/CommandWithDestination.java | 3 +-
.../apache/hadoop/fs/shell/MoveCommands.java | 6 +-
.../java/org/apache/hadoop/fs/shell/Touch.java | 3 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 13 +-
.../io/erasurecode/rawcoder/RSRawDecoder.java | 175 +++
.../io/erasurecode/rawcoder/RSRawDecoder2.java | 176 ---
.../io/erasurecode/rawcoder/RSRawEncoder.java | 75 ++
.../io/erasurecode/rawcoder/RSRawEncoder2.java | 76 --
.../rawcoder/RSRawErasureCoderFactory.java | 37 +
.../rawcoder/RSRawErasureCoderFactory2.java | 37 -
.../io/erasurecode/rawcoder/util/RSUtil.java | 149 ++-
.../io/erasurecode/rawcoder/util/RSUtil2.java | 172 ---
.../org/apache/hadoop/ipc/CallQueueManager.java | 134 ++-
.../main/java/org/apache/hadoop/ipc/Client.java | 166 +--
.../apache/hadoop/ipc/DecayRpcScheduler.java | 396 +++++--
.../hadoop/ipc/DecayRpcSchedulerMXBean.java | 2 +
.../apache/hadoop/ipc/DefaultRpcScheduler.java | 45 +
.../org/apache/hadoop/ipc/FairCallQueue.java | 45 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 8 +-
.../org/apache/hadoop/ipc/RpcScheduler.java | 8 +-
.../java/org/apache/hadoop/ipc/Schedulable.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 77 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 47 +-
.../hadoop/metrics2/lib/MutableQuantiles.java | 7 +-
.../hadoop/metrics2/util/QuantileEstimator.java | 32 +
.../hadoop/metrics2/util/SampleQuantiles.java | 2 +-
.../java/org/apache/hadoop/net/NetUtils.java | 31 +-
.../org/apache/hadoop/net/NetworkTopology.java | 17 +-
.../java/org/apache/hadoop/net/NodeBase.java | 18 +-
.../apache/hadoop/security/SecurityUtil.java | 53 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 42 +-
.../apache/hadoop/service/AbstractService.java | 2 +-
.../org/apache/hadoop/tracing/TraceUtils.java | 4 +-
.../hadoop/util/NativeLibraryChecker.java | 8 +-
.../main/java/org/apache/hadoop/util/Shell.java | 10 +-
.../apache/hadoop/util/ShutdownHookManager.java | 116 +-
.../hadoop/io/compress/lz4/Lz4Compressor.c | 4 +-
.../src/org/apache/hadoop/io/compress/lz4/lz4.c | 2 +-
.../hadoop/io/erasurecode/erasure_coder.c | 1 +
.../apache/hadoop/io/erasurecode/isal_load.c | 46 +-
.../apache/hadoop/io/erasurecode/isal_load.h | 6 +-
.../io/erasurecode/jni_erasure_code_native.c | 11 +-
.../src/main/resources/core-default.xml | 11 +-
.../src/site/markdown/ClusterSetup.md | 40 +-
.../src/site/markdown/CommandsManual.md | 2 +-
.../src/site/markdown/GroupsMapping.md | 26 +-
.../src/site/markdown/HttpAuthentication.md | 6 +-
.../src/site/markdown/UnixShellGuide.md | 2 +-
.../hadoop/crypto/key/TestValueQueue.java | 6 -
.../java/org/apache/hadoop/fs/TestFileUtil.java | 57 -
.../org/apache/hadoop/fs/TestFsShellCopy.java | 52 +-
.../org/apache/hadoop/fs/TestFsShellTouch.java | 88 ++
.../AbstractContractGetFileStatusTest.java | 23 +
.../coder/TestHHXORErasureCoder.java | 4 +-
.../erasurecode/coder/TestRSErasureCoder.java | 8 +-
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 33 +
.../erasurecode/rawcoder/TestRSRawCoder2.java | 33 -
.../apache/hadoop/ipc/TestCallQueueManager.java | 156 ++-
.../hadoop/ipc/TestDecayRpcScheduler.java | 42 +-
.../apache/hadoop/ipc/TestFairCallQueue.java | 79 +-
.../java/org/apache/hadoop/ipc/TestIPC.java | 101 +-
.../hadoop/ipc/TestIPCServerResponder.java | 30 +-
.../hadoop/ipc/TestIdentityProviders.java | 18 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 159 ++-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 192 +++-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 397 ++-----
.../org/apache/hadoop/net/TestNetUtils.java | 47 +-
.../hadoop/security/TestDoAsEffectiveUser.java | 6 +-
.../hadoop/security/TestSecurityUtil.java | 16 +-
.../hadoop/security/ssl/TestSSLFactory.java | 139 ++-
.../hadoop/util/TestShutdownHookManager.java | 57 +-
.../hadoop-common/src/test/proto/test.proto | 9 +
.../src/test/proto/test_rpc_service.proto | 4 +
...apache.hadoop.security.token.TokenIdentifier | 2 +-
.../scripts/hadoop-functions_test_helper.bash | 3 +-
.../scripts/hadoop_add_to_classpath_tools.bats | 47 +
.../hadoop_add_to_classpath_toolspath.bats | 74 --
.../src/test/scripts/hadoop_basic_init.bats | 5 +-
.../src/test/scripts/hadoop_bootstrap.bats | 19 +-
.../src/test/scripts/hadoop_confdir.bats | 24 +-
.../src/test/scripts/hadoop_entry_tests.bats | 49 +
.../src/test/scripts/hadoop_finalize.bats | 2 +-
.../src/test/scripts/hadoop_shellprofile.bats | 7 +
.../hadoop-kms/src/main/conf/kms-env.sh | 4 +-
.../hadoop/crypto/key/kms/server/KMS.java | 14 +-
.../hadoop-kms/src/main/libexec/kms-config.sh | 8 +-
.../hadoop-kms/src/main/sbin/kms.sh | 4 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 42 +
hadoop-dist/pom.xml | 18 +
.../dev-support/findbugsExcludeFile.xml | 1 +
.../org/apache/hadoop/hdfs/BlockReader.java | 10 +-
.../apache/hadoop/hdfs/BlockReaderFactory.java | 7 +-
.../apache/hadoop/hdfs/BlockReaderLocal.java | 10 +-
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 10 +-
.../org/apache/hadoop/hdfs/ClientContext.java | 56 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 26 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 30 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 3 -
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 6 +-
.../apache/hadoop/hdfs/ExternalBlockReader.java | 10 +-
.../apache/hadoop/hdfs/FileChecksumHelper.java | 187 +++-
.../apache/hadoop/hdfs/RemoteBlockReader.java | 29 +-
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 29 +-
.../org/apache/hadoop/hdfs/ReplicaAccessor.java | 7 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 2 +-
.../org/apache/hadoop/hdfs/protocol/Block.java | 4 +-
.../hdfs/protocol/DatanodeAdminProperties.java | 100 ++
.../apache/hadoop/hdfs/protocol/DatanodeID.java | 6 +
.../hadoop/hdfs/protocol/StripedBlockInfo.java | 61 +
.../datatransfer/DataTransferProtocol.java | 16 +-
.../hadoop/hdfs/protocol/datatransfer/Op.java | 1 +
.../hdfs/protocol/datatransfer/Sender.java | 19 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 42 +-
.../hdfs/util/CombinedHostsFileReader.java | 76 ++
.../hdfs/util/CombinedHostsFileWriter.java | 69 ++
.../hadoop/hdfs/util/StripedBlockUtil.java | 12 +
.../src/main/proto/datatransfer.proto | 9 +-
.../src/main/conf/httpfs-env.sh | 4 +-
.../src/main/libexec/httpfs-config.sh | 8 +-
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 4 +-
.../src/main/native/fuse-dfs/doc/README | 6 +-
.../src/main/native/fuse-dfs/fuse_connect.c | 5 +
.../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 12 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
.../src/main/bin/distribute-exclude.sh | 4 +-
.../hadoop-hdfs/src/main/bin/hdfs | 9 +-
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 6 +-
.../src/main/bin/refresh-namenodes.sh | 4 +-
.../hadoop-hdfs/src/main/bin/start-balancer.sh | 4 +-
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 4 +-
.../src/main/bin/start-secure-dns.sh | 4 +-
.../hadoop-hdfs/src/main/bin/stop-balancer.sh | 4 +-
.../hadoop-hdfs/src/main/bin/stop-dfs.sh | 4 +-
.../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 4 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +-
.../hdfs/protocol/datatransfer/Receiver.java | 28 +
.../hadoop/hdfs/server/balancer/Balancer.java | 2 +-
.../server/blockmanagement/BlockManager.java | 268 ++---
.../CombinedHostFileManager.java | 250 +++++
.../server/blockmanagement/DatanodeManager.java | 59 +-
.../blockmanagement/DatanodeStorageInfo.java | 10 +
.../blockmanagement/DecommissionManager.java | 30 +-
.../blockmanagement/HostConfigManager.java | 80 ++
.../server/blockmanagement/HostFileManager.java | 147 +--
.../hdfs/server/blockmanagement/HostSet.java | 114 ++
.../blockmanagement/LowRedundancyBlocks.java | 458 ++++++++
.../blockmanagement/UnderReplicatedBlocks.java | 448 --------
.../hdfs/server/datanode/BPServiceActor.java | 5 +
.../server/datanode/BlockChecksumHelper.java | 284 ++++-
.../hdfs/server/datanode/BlockReceiver.java | 4 +-
.../hdfs/server/datanode/BlockSender.java | 6 +-
.../server/datanode/DataNodeFaultInjector.java | 2 +
.../hdfs/server/datanode/DataXceiver.java | 43 +
.../hdfs/server/datanode/DataXceiverServer.java | 2 +-
.../datanode/IncrementalBlockReportManager.java | 9 +
.../hdfs/server/datanode/ReplicaInPipeline.java | 53 +-
.../erasurecode/ErasureCodingWorker.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 158 ++-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 6 +
.../datanode/fsdataset/impl/ReplicaMap.java | 2 +-
.../server/namenode/EncryptionZoneManager.java | 15 +
.../server/namenode/FSDirEncryptionZoneOp.java | 83 ++
.../hdfs/server/namenode/FSDirWriteFileOp.java | 4 +
.../hdfs/server/namenode/FSDirectory.java | 48 +-
.../hdfs/server/namenode/FSNamesystem.java | 27 +
.../hadoop/hdfs/server/namenode/NameNode.java | 9 +-
.../hdfs/server/namenode/NamenodeFsck.java | 2 +-
.../OfflineImageReconstructor.java | 312 +++---
.../offlineImageViewer/PBImageXmlWriter.java | 463 +++++---
.../src/main/resources/hdfs-default.xml | 41 +
.../src/main/webapps/hdfs/dfshealth.html | 13 +-
.../src/main/webapps/hdfs/explorer.js | 33 +-
.../hadoop-hdfs/src/site/markdown/Federation.md | 18 +-
.../src/site/markdown/HDFSCommands.md | 159 ++-
.../markdown/HDFSHighAvailabilityWithNFS.md | 4 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 4 +-
.../src/site/markdown/HdfsNfsGateway.md | 8 +-
.../src/site/markdown/HdfsUserGuide.md | 6 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 61 +-
.../org/apache/hadoop/TestRefreshCallQueue.java | 2 +-
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +-
.../hadoop/hdfs/TestBlockReaderLocal.java | 4 +-
.../hadoop/hdfs/TestBlocksScheduledCounter.java | 51 +
.../TestClientProtocolForPipelineRecovery.java | 53 +
.../org/apache/hadoop/hdfs/TestConnCache.java | 2 -
.../org/apache/hadoop/hdfs/TestDFSShell.java | 31 +
.../apache/hadoop/hdfs/TestDatanodeReport.java | 57 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 62 ++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 4 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 36 +-
.../hadoop/hdfs/TestExternalBlockReader.java | 8 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 247 +++++
.../datatransfer/sasl/TestSaslDataTransfer.java | 48 +
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 20 +-
.../TestBlocksWithNotEnoughRacks.java | 34 +-
.../blockmanagement/TestDatanodeManager.java | 8 +-
.../blockmanagement/TestHostFileManager.java | 10 +-
.../TestLowRedundancyBlockQueues.java | 182 +++
.../blockmanagement/TestPendingReplication.java | 14 +-
.../blockmanagement/TestReplicationPolicy.java | 158 +--
.../TestUnderReplicatedBlockQueues.java | 182 ---
.../server/datanode/TestBPOfferService.java | 96 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 137 ++-
.../hdfs/server/datanode/TestDataNodeUUID.java | 52 +
.../server/datanode/TestFsDatasetCache.java | 5 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 66 ++
.../namenode/TestDecommissioningStatus.java | 12 +-
.../hdfs/server/namenode/TestHostsFiles.java | 70 +-
.../hdfs/server/namenode/TestMetaSave.java | 2 +-
.../server/namenode/TestNameNodeMXBean.java | 25 +-
.../namenode/TestProtectedDirectories.java | 44 +
.../hdfs/server/namenode/TestStartup.java | 54 +-
.../TestUpgradeDomainBlockPlacementPolicy.java | 169 +++
.../namenode/ha/TestStandbyCheckpoints.java | 5 +
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../hadoop/hdfs/util/HostsFileWriter.java | 122 ++
.../hdfs/util/TestCombinedHostsFileReader.java | 79 ++
.../apache/hadoop/net/TestNetworkTopology.java | 7 +
.../apache/hadoop/tracing/TestTraceAdmin.java | 2 +-
.../TestTracingShortCircuitLocalRead.java | 4 +-
.../src/test/resources/dfs.hosts.json | 5 +
.../src/test/resources/testHDFSConf.xml | 6 +-
hadoop-mapreduce-project/bin/mapred | 14 +-
hadoop-mapreduce-project/bin/mapred-config.sh | 6 +-
.../bin/mr-jobhistory-daemon.sh | 4 +-
.../org/apache/hadoop/mapred/ClusterStatus.java | 35 +-
.../apache/hadoop/mapred/pipes/Submitter.java | 2 +-
.../org/apache/hadoop/mapreduce/TaskID.java | 57 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 48 +-
.../hadoop/mapreduce/util/ProcessTree.java | 7 +-
.../src/site/markdown/EncryptedShuffle.md | 2 +-
.../org/apache/hadoop/mapreduce/TestTaskID.java | 461 ++++++++
.../apache/hadoop/mapreduce/tools/TestCLI.java | 38 +-
.../mapreduce/v2/hs/TestHistoryFileManager.java | 5 +-
.../org/apache/hadoop/mapred/NotRunningJob.java | 7 +-
.../java/org/apache/hadoop/fs/DFSCIOTest.java | 10 +-
.../org/apache/hadoop/fs/TestFileSystem.java | 20 +-
.../java/org/apache/hadoop/fs/TestJHLA.java | 6 +-
.../java/org/apache/hadoop/hdfs/NNBench.java | 240 ++--
.../org/apache/hadoop/hdfs/TestNNBench.java | 84 ++
.../io/TestSequenceFileMergeProgress.java | 12 +-
.../hadoop/mapred/ClusterMapReduceTestCase.java | 14 +-
.../apache/hadoop/mapred/ReliabilityTest.java | 2 +-
.../apache/hadoop/mapred/TestAuditLogger.java | 9 +-
.../apache/hadoop/mapred/TestBadRecords.java | 8 +-
.../mapred/TestClusterMapReduceTestCase.java | 10 +
.../org/apache/hadoop/mapred/TestCollect.java | 10 +-
.../mapred/TestCommandLineJobSubmission.java | 9 +-
.../hadoop/mapred/TestFieldSelection.java | 11 +-
.../mapred/TestFileInputFormatPathFilter.java | 19 +-
.../apache/hadoop/mapred/TestGetSplitHosts.java | 7 +-
.../apache/hadoop/mapred/TestIFileStreams.java | 13 +-
.../org/apache/hadoop/mapred/TestInputPath.java | 7 +-
.../hadoop/mapred/TestJavaSerialization.java | 10 +-
.../org/apache/hadoop/mapred/TestJobName.java | 6 +
.../hadoop/mapred/TestJobSysDirWithDFS.java | 10 +-
.../mapred/TestKeyValueTextInputFormat.java | 15 +-
.../apache/hadoop/mapred/TestLazyOutput.java | 7 +-
.../hadoop/mapred/TestMRCJCFileInputFormat.java | 32 +-
.../mapred/TestMRCJCFileOutputCommitter.java | 28 +-
.../mapred/TestMRTimelineEventHandling.java | 19 +-
.../apache/hadoop/mapred/TestMapProgress.java | 9 +-
.../org/apache/hadoop/mapred/TestMerge.java | 7 +-
.../apache/hadoop/mapred/TestMiniMRBringup.java | 6 +-
.../hadoop/mapred/TestMiniMRChildTask.java | 16 +-
.../hadoop/mapred/TestMiniMRDFSCaching.java | 14 +-
.../hadoop/mapred/TestMultiFileInputFormat.java | 19 +-
.../hadoop/mapred/TestMultiFileSplit.java | 10 +-
.../hadoop/mapred/TestMultipleLevelCaching.java | 12 +-
.../mapred/TestMultipleTextOutputFormat.java | 23 +-
.../apache/hadoop/mapred/TestNetworkedJob.java | 89 +-
.../hadoop/mapred/TestOldCombinerGrouping.java | 5 +-
.../apache/hadoop/mapred/TestReduceFetch.java | 10 +-
.../mapred/TestReduceFetchFromPartialMem.java | 46 +-
.../apache/hadoop/mapred/TestReduceTask.java | 18 +-
.../TestSequenceFileAsBinaryInputFormat.java | 19 +-
.../TestSequenceFileAsBinaryOutputFormat.java | 31 +-
.../TestSequenceFileAsTextInputFormat.java | 33 +-
.../mapred/TestSequenceFileInputFilter.java | 32 +-
.../mapred/TestSequenceFileInputFormat.java | 29 +-
.../apache/hadoop/mapred/TestSortedRanges.java | 19 +-
.../TestSpecialCharactersInOutputPath.java | 21 +-
.../hadoop/mapred/TestStatisticsCollector.java | 10 +-
.../hadoop/mapred/TestUserDefinedCounters.java | 24 +-
.../hadoop/mapred/TestWritableJobConf.java | 20 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 8 +-
.../hadoop/mapred/join/TestDatamerge.java | 42 +-
.../hadoop/mapred/join/TestTupleWritable.java | 24 +-
.../TestWrappedRecordReaderClassloader.java | 7 +-
.../mapred/lib/TestDelegatingInputFormat.java | 9 +-
.../hadoop/mapred/lib/TestLineInputFormat.java | 7 +-
.../hadoop/mapred/lib/TestMultipleInputs.java | 2 -
.../mapred/lib/aggregate/TestAggregates.java | 14 +-
.../mapred/lib/db/TestConstructQuery.java | 16 +-
.../apache/hadoop/mapred/pipes/TestPipes.java | 9 +-
.../hadoop/mapreduce/TestLocalRunner.java | 34 +-
.../hadoop/mapreduce/TestMRJobClient.java | 49 +-
.../mapreduce/TestMapReduceLazyOutput.java | 9 +-
.../mapreduce/TestNewCombinerGrouping.java | 5 +-
.../hadoop/mapreduce/TestValueIterReset.java | 8 +-
.../TestYarnClientProtocolProvider.java | 5 +-
.../lib/aggregate/TestMapReduceAggregates.java | 28 +-
.../mapreduce/lib/db/TestDBOutputFormat.java | 17 +-
.../mapreduce/lib/db/TestIntegerSplitter.java | 15 +-
.../mapreduce/lib/db/TestTextSplitter.java | 18 +-
.../lib/fieldsel/TestMRFieldSelection.java | 20 +-
.../lib/input/TestCombineFileInputFormat.java | 4 +-
.../TestMRSequenceFileAsBinaryInputFormat.java | 21 +-
.../TestMRSequenceFileAsTextInputFormat.java | 27 +-
.../input/TestMRSequenceFileInputFilter.java | 39 +-
.../lib/input/TestNLineInputFormat.java | 34 +-
.../mapreduce/lib/join/TestJoinDatamerge.java | 52 +-
.../mapreduce/lib/join/TestJoinProperties.java | 44 +-
.../lib/join/TestJoinTupleWritable.java | 24 +-
.../lib/join/TestWrappedRRClassloader.java | 17 +-
.../TestMRSequenceFileAsBinaryOutputFormat.java | 35 +-
.../lib/partition/TestBinaryPartitioner.java | 16 +-
.../lib/partition/TestKeyFieldHelper.java | 9 +-
.../TestMRKeyFieldBasedPartitioner.java | 6 +-
.../partition/TestTotalOrderPartitioner.java | 11 +-
.../mapreduce/util/TestMRAsyncDiskService.java | 15 +-
.../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +-
.../mapreduce/v2/TestMiniMRProxyUser.java | 30 +-
.../hadoop/mapreduce/v2/TestNonExistentJob.java | 18 +-
.../apache/hadoop/examples/TestWordStats.java | 15 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 17 +
.../apache/hadoop/tools/HadoopArchiveLogs.java | 4 +-
.../hadoop/tools/TestHadoopArchiveLogs.java | 4 +-
hadoop-tools/hadoop-archives/pom.xml | 17 +
hadoop-tools/hadoop-aws/pom.xml | 17 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 77 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 12 +-
.../s3a/TestS3AContractGetFileStatus.java | 31 +
.../src/test/resources/contract/s3a.xml | 5 +
hadoop-tools/hadoop-azure/pom.xml | 18 +
.../hadoop-azure/src/site/markdown/index.md | 3 +
hadoop-tools/hadoop-datajoin/pom.xml | 16 +
.../apache/hadoop/contrib/utils/join/README.txt | 2 +-
hadoop-tools/hadoop-distcp/pom.xml | 11 +
.../java/org/apache/hadoop/tools/DistCp.java | 79 --
.../apache/hadoop/tools/DistCpConstants.java | 16 -
.../apache/hadoop/tools/DistCpOptionSwitch.java | 9 -
.../org/apache/hadoop/tools/DistCpOptions.java | 22 -
.../org/apache/hadoop/tools/OptionsParser.java | 5 -
.../apache/hadoop/tools/mapred/CopyMapper.java | 68 --
.../src/site/markdown/DistCp.md.vm | 39 -
.../apache/hadoop/tools/TestOptionsParser.java | 17 +-
hadoop-tools/hadoop-extras/pom.xml | 17 +
hadoop-tools/hadoop-gridmix/pom.xml | 17 +
hadoop-tools/hadoop-kafka/pom.xml | 17 +
hadoop-tools/hadoop-openstack/pom.xml | 17 +
.../hadoop-openstack/src/site/markdown/index.md | 7 +-
.../native/pipes/debug/pipes-default-script | 5 +-
hadoop-tools/hadoop-rumen/pom.xml | 17 +
hadoop-tools/hadoop-sls/pom.xml | 17 +
.../hadoop-sls/src/main/bin/rumen2sls.sh | 6 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 10 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 9 +
hadoop-tools/hadoop-streaming/pom.xml | 17 +
.../apache/hadoop/streaming/DumpTypedBytes.java | 2 +-
.../hadoop/streaming/HadoopStreaming.java | 2 +-
.../apache/hadoop/streaming/LoadTypedBytes.java | 2 +-
.../org/apache/hadoop/streaming/StreamJob.java | 16 +-
.../streaming/TestStreamingBadRecords.java | 9 +-
.../hadoop-yarn/bin/start-yarn.sh | 4 +-
.../hadoop-yarn/bin/stop-yarn.sh | 4 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 8 +-
.../hadoop-yarn/bin/yarn-config.sh | 6 +-
.../hadoop-yarn/bin/yarn-daemon.sh | 4 +-
.../hadoop-yarn/bin/yarn-daemons.sh | 4 +-
.../yarn/api/records/ResourceUtilization.java | 22 +
.../hadoop/yarn/conf/YarnConfiguration.java | 15 +
.../distributedshell/ApplicationMaster.java | 13 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 3 +-
.../hadoop/yarn/client/TestRMFailover.java | 9 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 413 +++++++
.../yarn/client/api/impl/TestYarnClient.java | 18 +
.../api/impl/FileSystemTimelineWriter.java | 112 +-
.../apache/hadoop/yarn/util/ConverterUtils.java | 16 +-
.../yarn/util/ProcfsBasedProcessTree.java | 6 +-
.../util/ResourceCalculatorProcessTree.java | 8 +-
.../apache/hadoop/yarn/util/SystemClock.java | 5 +-
.../yarn/util/WindowsBasedProcessTree.java | 5 +-
.../hadoop/yarn/util/resource/Resources.java | 2 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 22 +
.../src/main/resources/yarn-default.xml | 26 +-
.../api/records/TestResourceUtilization.java | 63 ++
.../yarn/util/TestProcfsBasedProcessTree.java | 8 -
.../util/TestResourceCalculatorProcessTree.java | 3 +-
.../yarn/util/TestWindowsBasedProcessTree.java | 3 +-
.../ApplicationHistoryServer.java | 15 +-
.../hadoop/yarn/server/webapp/WebPageUtils.java | 6 +
.../hadoop/yarn/server/webapp/WebServices.java | 22 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 5 +
.../nodemanager/NodeStatusUpdaterImpl.java | 50 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 14 +-
.../amrmproxy/DefaultRequestInterceptor.java | 7 +
.../containermanager/ContainerManagerImpl.java | 49 +-
.../logaggregation/AppLogAggregatorImpl.java | 14 +-
.../monitor/ContainerMetrics.java | 69 ++
.../nodemanager/webapp/NMWebServices.java | 6 +-
.../TestDockerContainerExecutorWithMocks.java | 2 +-
.../containermanager/TestNMProxy.java | 3 +-
.../TestResourceLocalizationService.java | 2 +-
.../TestLogAggregationService.java | 12 +-
.../monitor/TestContainerMetrics.java | 58 +-
.../webapp/TestNMWebServicesApps.java | 9 +-
.../server/resourcemanager/AdminService.java | 40 +-
.../server/resourcemanager/ClientRMService.java | 9 +-
.../resourcemanager/NodesListManager.java | 104 +-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../resourcemanager/ResourceTrackerService.java | 88 +-
.../monitor/SchedulingEditPolicy.java | 8 +-
.../monitor/SchedulingMonitor.java | 4 -
.../CapacitySchedulerPreemptionContext.java | 52 +
.../CapacitySchedulerPreemptionUtils.java | 65 ++
.../capacity/FifoCandidatesSelector.java | 364 ++++++
.../capacity/PreemptableResourceCalculator.java | 370 +++++++
.../capacity/PreemptionCandidatesSelector.java | 52 +
.../ProportionalCapacityPreemptionPolicy.java | 1044 ++++--------------
.../monitor/capacity/TempQueuePerPartition.java | 159 +++
.../resource/DynamicResourceConfiguration.java | 13 +-
.../rmcontainer/RMContainer.java | 1 +
.../server/resourcemanager/rmnode/RMNode.java | 4 +
.../resourcemanager/rmnode/RMNodeImpl.java | 22 +-
.../scheduler/AbstractYarnScheduler.java | 170 +--
.../scheduler/ClusterNodeTracker.java | 300 +++++
.../resourcemanager/scheduler/NodeFilter.java | 33 +
.../scheduler/PreemptableResourceScheduler.java | 2 +-
.../server/resourcemanager/scheduler/Queue.java | 20 +
.../scheduler/ResourceLimits.java | 9 +
.../scheduler/SchedulerNode.java | 9 +-
.../scheduler/capacity/AbstractCSQueue.java | 71 +-
.../scheduler/capacity/CSAssignment.java | 11 +
.../scheduler/capacity/CSQueueUtils.java | 13 +
.../scheduler/capacity/CapacityScheduler.java | 216 ++--
.../CapacitySchedulerConfiguration.java | 61 +-
.../capacity/CapacitySchedulerContext.java | 15 +-
.../scheduler/capacity/LeafQueue.java | 85 +-
.../scheduler/capacity/ParentQueue.java | 157 ++-
.../scheduler/capacity/QueueCapacities.java | 38 +-
.../allocator/AbstractContainerAllocator.java | 2 +
.../capacity/allocator/ContainerAllocation.java | 12 +
.../allocator/RegularContainerAllocator.java | 39 +-
.../capacity/preemption/KillableContainer.java | 45 +
.../capacity/preemption/PreemptableQueue.java | 96 ++
.../capacity/preemption/PreemptionManager.java | 165 +++
.../scheduler/common/AssignmentInformation.java | 6 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 32 +-
.../common/fica/FiCaSchedulerNode.java | 65 +-
.../scheduler/event/SchedulerEventType.java | 15 +-
.../scheduler/fair/FSAppAttempt.java | 13 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 8 +
.../scheduler/fair/FairScheduler.java | 122 +-
.../scheduler/fifo/FifoScheduler.java | 44 +-
.../resourcemanager/webapp/AppAttemptPage.java | 4 +-
.../webapp/CapacitySchedulerPage.java | 16 +-
.../resourcemanager/webapp/NodesPage.java | 53 +-
.../webapp/RMAppAttemptBlock.java | 61 +-
.../resourcemanager/webapp/RMWebAppFilter.java | 40 +-
.../resourcemanager/webapp/RMWebServices.java | 32 +-
.../webapp/dao/CapacitySchedulerInfo.java | 26 +-
.../webapp/dao/ReservationIdInfo.java | 64 --
.../webapp/dao/ReservationInfo.java | 7 +-
.../yarn/server/resourcemanager/MockNodes.java | 9 +
.../yarn/server/resourcemanager/MockRM.java | 7 +-
.../ParameterizedSchedulerTestBase.java | 36 +-
.../yarn/server/resourcemanager/TestRM.java | 4 -
.../resourcemanager/TestRMAdminService.java | 77 +-
.../resourcemanager/TestRMDispatcher.java | 4 +-
.../server/resourcemanager/TestRMRestart.java | 8 +-
.../TestResourceTrackerService.java | 222 +++-
.../TestWorkPreservingRMRestart.java | 9 -
.../applicationsmanager/TestAMRestart.java | 7 +-
...estProportionalCapacityPreemptionPolicy.java | 137 +--
...pacityPreemptionPolicyForNodePartitions.java | 80 +-
.../recovery/TestZKRMStateStorePerf.java | 9 +-
.../reservation/TestReservationSystem.java | 4 -
.../rmapp/TestNodesListManager.java | 5 +-
.../scheduler/TestAbstractYarnScheduler.java | 20 +-
.../capacity/TestApplicationLimits.java | 2 +
.../capacity/TestApplicationPriority.java | 6 +-
.../capacity/TestCapacityScheduler.java | 8 +-
.../TestCapacitySchedulerPreemption.java | 683 ++++++++++++
.../scheduler/capacity/TestChildQueueOrder.java | 2 +
.../capacity/TestContainerAllocation.java | 13 +-
.../scheduler/capacity/TestLeafQueue.java | 3 +
.../TestNodeLabelContainerAllocation.java | 176 ++-
.../scheduler/capacity/TestParentQueue.java | 2 +
.../scheduler/capacity/TestQueueCapacities.java | 4 +-
.../scheduler/capacity/TestReservations.java | 21 +-
.../scheduler/capacity/TestUtils.java | 36 +
.../scheduler/fair/FairSchedulerTestBase.java | 22 +-
.../fair/TestContinuousScheduling.java | 78 +-
.../scheduler/fair/TestFairScheduler.java | 82 +-
.../fair/TestFairSchedulerPreemption.java | 2 +-
.../scheduler/fifo/TestFifoScheduler.java | 19 +-
.../security/TestClientToAMTokens.java | 4 -
.../security/TestDelegationTokenRenewer.java | 3 -
.../resourcemanager/webapp/TestNodesPage.java | 37 +-
.../webapp/TestRMWebServicesApps.java | 24 +-
.../TestRMWebServicesAppsModification.java | 10 +-
.../webapp/TestRMWebServicesCapacitySched.java | 3 +
.../webapp/TestRMWebServicesNodes.java | 12 +-
.../webapp/TestRMWebServicesReservation.java | 15 +-
.../src/test/resources/test-fair-scheduler.xml | 34 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 144 ++-
.../yarn/server/TestMiniYARNClusterForHA.java | 4 -
.../src/site/markdown/CapacityScheduler.md | 22 +-
.../site/markdown/DockerContainerExecutor.md.vm | 2 +-
.../src/site/markdown/FairScheduler.md | 6 +-
.../src/site/markdown/ResourceManagerRest.md | 217 ++++
.../src/site/markdown/SecureContainer.md | 2 +-
.../src/site/markdown/TimelineServer.md | 8 +-
.../src/site/markdown/YarnCommands.md | 45 +-
541 files changed, 16565 insertions(+), 6900 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48a8c9c3/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48a8c9c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
[24/50] [abbrv] hadoop git commit: HADOOP-11687. Ignore x-* and
response headers when copying an Amazon S3 object. Contributed by Aaron
Peterson and harsh.
Posted by ae...@apache.org.
HADOOP-11687. Ignore x-* and response headers when copying an Amazon S3 object. Contributed by Aaron Peterson and harsh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/256c82fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/256c82fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/256c82fe
Branch: refs/heads/HDFS-1312
Commit: 256c82fe2981748cd0befc5490d8118d139908f9
Parents: 3488c4f
Author: Harsh J <ha...@cloudera.com>
Authored: Fri Apr 1 14:18:10 2016 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Fri Apr 1 14:18:10 2016 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 70 +++++++++++++++++++-
.../src/site/markdown/tools/hadoop-aws/index.md | 7 ++
2 files changed, 76 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256c82fe/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 7ab6c79..6afb05d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -26,6 +26,7 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -1128,7 +1129,7 @@ public class S3AFileSystem extends FileSystem {
}
ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
- final ObjectMetadata dstom = srcom.clone();
+ ObjectMetadata dstom = cloneObjectMetadata(srcom);
if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
dstom.setSSEAlgorithm(serverSideEncryptionAlgorithm);
}
@@ -1235,6 +1236,73 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Creates a copy of the passed {@link ObjectMetadata}.
+ * Does so without using the {@link ObjectMetadata#clone()} method,
+ * to avoid copying unnecessary headers.
+ * @param source the {@link ObjectMetadata} to copy
+ * @return a copy of {@link ObjectMetadata} with only relevant attributes
+ */
+ private ObjectMetadata cloneObjectMetadata(ObjectMetadata source) {
+ // This approach may be too brittle, especially if
+ // in future there are new attributes added to ObjectMetadata
+ // that we do not explicitly call to set here
+ ObjectMetadata ret = new ObjectMetadata();
+
+ // Non null attributes
+ ret.setContentLength(source.getContentLength());
+
+ // Possibly null attributes
+ // Allowing nulls to pass breaks it during later use
+ if (source.getCacheControl() != null) {
+ ret.setCacheControl(source.getCacheControl());
+ }
+ if (source.getContentDisposition() != null) {
+ ret.setContentDisposition(source.getContentDisposition());
+ }
+ if (source.getContentEncoding() != null) {
+ ret.setContentEncoding(source.getContentEncoding());
+ }
+ if (source.getContentMD5() != null) {
+ ret.setContentMD5(source.getContentMD5());
+ }
+ if (source.getContentType() != null) {
+ ret.setContentType(source.getContentType());
+ }
+ if (source.getExpirationTime() != null) {
+ ret.setExpirationTime(source.getExpirationTime());
+ }
+ if (source.getExpirationTimeRuleId() != null) {
+ ret.setExpirationTimeRuleId(source.getExpirationTimeRuleId());
+ }
+ if (source.getHttpExpiresDate() != null) {
+ ret.setHttpExpiresDate(source.getHttpExpiresDate());
+ }
+ if (source.getLastModified() != null) {
+ ret.setLastModified(source.getLastModified());
+ }
+ if (source.getOngoingRestore() != null) {
+ ret.setOngoingRestore(source.getOngoingRestore());
+ }
+ if (source.getRestoreExpirationTime() != null) {
+ ret.setRestoreExpirationTime(source.getRestoreExpirationTime());
+ }
+ if (source.getSSEAlgorithm() != null) {
+ ret.setSSEAlgorithm(source.getSSEAlgorithm());
+ }
+ if (source.getSSECustomerAlgorithm() != null) {
+ ret.setSSECustomerAlgorithm(source.getSSECustomerAlgorithm());
+ }
+ if (source.getSSECustomerKeyMd5() != null) {
+ ret.setSSECustomerKeyMd5(source.getSSECustomerKeyMd5());
+ }
+
+ for (Map.Entry<String, String> e : source.getUserMetadata().entrySet()) {
+ ret.addUserMetadata(e.getKey(), e.getValue());
+ }
+ return ret;
+ }
+
+ /**
* Return the number of bytes that large input files should be optimally
* be split into to minimize i/o time.
* @deprecated use {@link #getDefaultBlockSize(Path)} instead
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256c82fe/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index af3541f..7382029 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -417,6 +417,13 @@ which pass in authentication details to the test runner
These are both Hadoop XML configuration files, which must be placed into
`hadoop-tools/hadoop-aws/src/test/resources`.
+### `core-site.xml`
+
+This file pre-exists and sources the configurations created
+under `auth-keys.xml`.
+
+For most purposes you will not need to edit this file unless you
+need to apply a specific, non-default property change during the tests.
### `auth-keys.xml`
[38/50] [abbrv] hadoop git commit: HADOOP-12959. Add additional
github web site for ISA-L library (Li Bo via cmccabe)
Posted by ae...@apache.org.
HADOOP-12959. Add additional github web site for ISA-L library (Li Bo via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f65f5b18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f65f5b18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f65f5b18
Branch: refs/heads/HDFS-1312
Commit: f65f5b18fd4647e868b8d2a2c035a3b64dc16aa8
Parents: f61de41
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Apr 4 16:30:32 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Apr 4 16:30:32 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f65f5b18/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 408cae1..c7a91da 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -75,6 +75,7 @@ Optional packages:
$ sudo apt-get install snappy libsnappy-dev
* Intel ISA-L library for erasure coding
Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
+ (OR https://github.com/01org/isa-l)
* Bzip2
$ sudo apt-get install bzip2 libbz2-dev
* Jansson (C Library for JSON)
@@ -188,11 +189,12 @@ Maven build goals:
Intel ISA-L build options:
- Intel ISA-L is a erasure coding library that can be utilized by the native code.
+ Intel ISA-L is an erasure coding library that can be utilized by the native code.
It is currently an optional component, meaning that Hadoop can be built with
or without this dependency. Note the library is used via dynamic module. Please
reference the official site for the library details.
https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
+ (OR https://github.com/01org/isa-l)
* Use -Drequire.isal to fail the build if libisal.so is not found.
If this option is not specified and the isal library is missing,
[40/50] [abbrv] hadoop git commit: HDFS-9917. IBR accumulate more
objects when SNN was down for sometime. (Contributed by Brahma Reddy Battula)
Posted by ae...@apache.org.
HDFS-9917. IBR accumulate more objects when SNN was down for sometime. (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/818d6b79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/818d6b79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/818d6b79
Branch: refs/heads/HDFS-1312
Commit: 818d6b799eead13a17a0214172df60a269b046fb
Parents: f6b1a81
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Apr 5 09:49:39 2016 +0800
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Apr 5 09:49:39 2016 +0800
----------------------------------------------------------------------
.../hdfs/server/datanode/BPServiceActor.java | 5 +
.../datanode/IncrementalBlockReportManager.java | 9 ++
.../server/datanode/TestBPOfferService.java | 96 +++++++++++++++++++-
3 files changed, 107 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/818d6b79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 49f64c2..39f8219 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -798,6 +798,11 @@ class BPServiceActor implements Runnable {
// and re-register
register(nsInfo);
scheduler.scheduleHeartbeat();
+ // HDFS-9917,Standby NN IBR can be very huge if standby namenode is down
+ // for sometime.
+ if (state == HAServiceState.STANDBY) {
+ ibrManager.clearIBRs();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/818d6b79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
index b9b348a..e95142d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java
@@ -258,4 +258,13 @@ class IncrementalBlockReportManager {
}
}
}
+
+ void clearIBRs() {
+ pendingIBRs.clear();
+ }
+
+ @VisibleForTesting
+ int getPendingIBRSize() {
+ return pendingIBRs.size();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/818d6b79/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index 95a103e..29db702 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -30,6 +30,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -48,10 +49,12 @@ import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
@@ -92,7 +95,10 @@ public class TestBPOfferService {
private DatanodeProtocolClientSideTranslatorPB mockNN1;
private DatanodeProtocolClientSideTranslatorPB mockNN2;
- private final NNHAStatusHeartbeat[] mockHaStatuses = new NNHAStatusHeartbeat[2];
+ private final NNHAStatusHeartbeat[] mockHaStatuses =
+ new NNHAStatusHeartbeat[2];
+ private final DatanodeCommand[][] datanodeCommands =
+ new DatanodeCommand[2][0];
private final int[] heartbeatCounts = new int[2];
private DataNode mockDn;
private FsDatasetSpi<?> mockFSDataset;
@@ -147,6 +153,7 @@ public class TestBPOfferService {
Mockito.any(VolumeFailureSummary.class),
Mockito.anyBoolean());
mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0);
+ datanodeCommands[nnIdx] = new DatanodeCommand[0];
return mock;
}
@@ -165,9 +172,12 @@ public class TestBPOfferService {
@Override
public HeartbeatResponse answer(InvocationOnMock invocation) throws Throwable {
heartbeatCounts[nnIdx]++;
- return new HeartbeatResponse(new DatanodeCommand[0],
- mockHaStatuses[nnIdx], null,
+ HeartbeatResponse heartbeatResponse = new HeartbeatResponse(
+ datanodeCommands[nnIdx], mockHaStatuses[nnIdx], null,
ThreadLocalRandom.current().nextLong() | 1L);
+ //reset the command
+ datanodeCommands[nnIdx] = new DatanodeCommand[0];
+ return heartbeatResponse;
}
}
@@ -709,4 +719,84 @@ public class TestBPOfferService {
bpos.join();
}
}
+
+ /*
+ * HDFS-9917 : Standby IBR accumulation when Standby was down.
+ */
+ @Test
+ public void testIBRClearanceForStandbyOnReRegister() throws Exception {
+ final BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
+ bpos.start();
+ try {
+ waitForInitialization(bpos);
+ // Should start with neither NN as active.
+ assertNull(bpos.getActiveNN());
+ // Have NN1 claim active at txid 1
+ mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
+ bpos.triggerHeartbeatForTests();
+ // Now mockNN1 is acting like active namenode and mockNN2 as Standby
+ assertSame(mockNN1, bpos.getActiveNN());
+ // Return nothing when active Active Namenode gets IBRs
+ Mockito.doNothing().when(mockNN1).blockReceivedAndDeleted(
+ Mockito.any(DatanodeRegistration.class), Mockito.anyString(), Mockito
+ .any(StorageReceivedDeletedBlocks[].class));
+
+ final IOException re = new IOException(
+ "Standby NN is currently not able to process IBR");
+
+ final AtomicBoolean ibrReported = new AtomicBoolean(false);
+ // throw exception for standby when first IBR is receieved
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ ibrReported.set(true);
+ throw re;
+ }
+ }).when(mockNN2).blockReceivedAndDeleted(
+ Mockito.any(DatanodeRegistration.class), Mockito.anyString(), Mockito
+ .any(StorageReceivedDeletedBlocks[].class));
+
+ DatanodeStorage storage = Mockito.mock(DatanodeStorage.class);
+ Mockito.doReturn(storage).when(mockFSDataset).getStorage("storage0");
+ // Add IBRs
+ bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, null, "storage0", false);
+ // Send heartbeat so that the BpServiceActor can send IBR to
+ // namenode
+ bpos.triggerHeartbeatForTests();
+ // Wait till first IBR is received at standbyNN. Just for confirmation.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return ibrReported.get();
+ }
+ }, 100, 1000);
+
+ // Send register command back to Datanode to reRegister().
+ // After reRegister IBRs should be cleared.
+ datanodeCommands[1] = new DatanodeCommand[] { new RegisterCommand() };
+ assertEquals(
+ "IBR size before reRegister should be non-0", 1, getStandbyIBRSize(
+ bpos));
+ bpos.triggerHeartbeatForTests();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return getStandbyIBRSize(bpos) == 0;
+ }
+ }, 100, 1000);
+ } finally {
+ bpos.stop();
+ bpos.join();
+ }
+ }
+
+ private int getStandbyIBRSize(BPOfferService bpos) {
+ List<BPServiceActor> bpServiceActors = bpos.getBPServiceActors();
+ for (BPServiceActor bpServiceActor : bpServiceActors) {
+ if (bpServiceActor.state == HAServiceState.STANDBY) {
+ return bpServiceActor.getIbrManager().getPendingIBRSize();
+ }
+ }
+ return -1;
+ }
}
[11/50] [abbrv] hadoop git commit: YARN-4857. Add missing default
configuration regarding preemption of CapacityScheduler. Contributed by Kai
Sasaki.
Posted by ae...@apache.org.
YARN-4857. Add missing default configuration regarding preemption of CapacityScheduler. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0064cba1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0064cba1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0064cba1
Branch: refs/heads/HDFS-1312
Commit: 0064cba169d1bb761f6e81ee86830be598d7c500
Parents: f1b8f6b
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Mar 31 14:05:49 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Mar 31 14:05:49 2016 +0530
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 58 ++++++++++++++++++++
1 file changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0064cba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index ea1afe4..33cd919 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -908,6 +908,64 @@
<value>600000</value>
</property>
+ <property>
+ <description>
+ If true, run the policy but do not affect the cluster with preemption and kill events.
+ </description>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.observe_only</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ Time in milliseconds between invocations of this ProportionalCapacityPreemptionPolicy
+ policy.
+ </description>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval</name>
+ <value>3000</value>
+ </property>
+
+ <property>
+ <description>
+ Time in milliseconds between requesting a preemption from an application and killing
+ the container.
+ </description>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill</name>
+ <value>15000</value>
+ </property>
+
+ <property>
+ <description>
+ Maximum percentage of resources preempted in a single round. By controlling this valueone
+ can throttle the pace at which containers are reclaimed from the cluster. After computing
+ the total desired preemption, the policy scales it back within this limit.
+ </description>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
+ <value>0.1</value>
+ </property>
+
+ <property>
+ <description>
+ Maximum amount of resources above the target capacity ignored for preemption.
+ This defines a deadzone around the target capacity that helps prevent thrashing and
+ oscillations around the computed target balance. High values would slow the time to capacity
+ and (absent natural.completions) it might prevent convergence to guaranteed capacity.
+ </description>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity</name>
+ <value>0.1</value>
+ </property>
+
+ <property>
+ <description>
+ Given a computed preemption target, account for containers naturally expiring and preempt
+ only this percentage of the delta. This determines the rate of geometric convergence into
+ the deadzone (MAX_IGNORED_OVER_CAPACITY). For example, a termination factor of 0.5 will reclaim
+ almost 95% of resources within 5 * #WAIT_TIME_BEFORE_KILL, even absent natural termination.
+ </description>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
+ <value>0.2</value>
+ </property>
+
<!-- Node Manager Configuration -->
<property>
[41/50] [abbrv] hadoop git commit: YARN-4880. Running
TestZKRMStateStorePerf with real zookeeper cluster throws NPE. Contributed by
Sunil G
Posted by ae...@apache.org.
YARN-4880. Running TestZKRMStateStorePerf with real zookeeper cluster throws NPE. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/552237d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/552237d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/552237d4
Branch: refs/heads/HDFS-1312
Commit: 552237d4a34ab10fa5f9ec7aad7942f2a110993e
Parents: 818d6b7
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Apr 5 14:25:32 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Apr 5 14:26:19 2016 +0530
----------------------------------------------------------------------
.../resourcemanager/recovery/TestZKRMStateStorePerf.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/552237d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
index 4b0b06a..bd25def 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStorePerf.java
@@ -91,7 +91,9 @@ public class TestZKRMStateStorePerf extends RMStateStoreTestBase
if (appTokenMgr != null) {
appTokenMgr.stop();
}
- curatorTestingServer.stop();
+ if (curatorTestingServer != null) {
+ curatorTestingServer.stop();
+ }
}
private void initStore(String hostPort) {
@@ -99,8 +101,9 @@ public class TestZKRMStateStorePerf extends RMStateStoreTestBase
RMContext rmContext = mock(RMContext.class);
conf = new YarnConfiguration();
- conf.set(YarnConfiguration.RM_ZK_ADDRESS,
- optHostPort.or(curatorTestingServer.getConnectString()));
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort
+ .or((curatorTestingServer == null) ? "" : curatorTestingServer
+ .getConnectString()));
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
store = new ZKRMStateStore();
[37/50] [abbrv] hadoop git commit: YARN-4706. UI Hosting
Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9)
Posted by ae...@apache.org.
YARN-4706. UI Hosting Configuration in TimelineServer doc is broken. (Akira AJISAKA via gtcarrera9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f61de417
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f61de417
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f61de417
Branch: refs/heads/HDFS-1312
Commit: f61de4173684aa1767cef20b3cb4d54df20273cd
Parents: a7d1fb0
Author: Li Lu <gt...@apache.org>
Authored: Mon Apr 4 14:39:47 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Mon Apr 4 14:40:27 2016 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/TimelineServer.md | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f61de417/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 9283e58..f20bd2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -56,6 +56,7 @@ With the introduction of the timeline server, the Application History Server bec
the Timeline Server.
Generic information includes application level data such as
+
* queue-name,
* user information and the like set in the `ApplicationSubmissionContext`,
* a list of application-attempts that ran for an application
@@ -192,6 +193,7 @@ selected if this policy is `HTTPS_ONLY`.
#### UI Hosting Configuration
The timeline service can host multiple UIs if enabled. The service can support both static web sites hosted in a directory or war files bundled. The web UI is then hosted on the timeline service HTTP port under the path configured.
+
| Configuration Property | Description |
|:---- |:---- |
| `yarn.timeline-service.ui-names` | Comma separated list of UIs that will be hosted. Defaults to `none`. |
[36/50] [abbrv] hadoop git commit: HDFS-10178. Permanent write
failures can happen if pipeline recoveries occur for the first packet.
Contributed by Kihwal Lee.
Posted by ae...@apache.org.
HDFS-10178. Permanent write failures can happen if pipeline recoveries occur for the first packet. Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7d1fb0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7d1fb0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7d1fb0c
Branch: refs/heads/HDFS-1312
Commit: a7d1fb0cd2fdbf830602eb4dbbd9bbe62f4d5584
Parents: 154d253
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Apr 4 16:39:23 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Apr 4 16:40:00 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockReceiver.java | 2 +
.../hdfs/server/datanode/BlockSender.java | 6 ++-
.../server/datanode/DataNodeFaultInjector.java | 2 +
.../TestClientProtocolForPipelineRecovery.java | 53 ++++++++++++++++++++
4 files changed, 62 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 2e4ee02..fb0c1c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -574,6 +574,8 @@ class BlockReceiver implements Closeable {
if (mirrorOut != null && !mirrorError) {
try {
long begin = Time.monotonicNow();
+ // For testing. Normally no-op.
+ DataNodeFaultInjector.get().stopSendingPacketDownstream();
packetReceiver.mirrorPacketTo(mirrorOut);
mirrorOut.flush();
long now = Time.monotonicNow();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 773a64c..398935d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -300,11 +300,15 @@ class BlockSender implements java.io.Closeable {
// The meta file will contain only the header if the NULL checksum
// type was used, or if the replica was written to transient storage.
+ // Also, when only header portion of a data packet was transferred
+ // and then pipeline breaks, the meta file can contain only the
+ // header and 0 byte in the block data file.
// Checksum verification is not performed for replicas on transient
// storage. The header is important for determining the checksum
// type later when lazy persistence copies the block to non-transient
// storage and computes the checksum.
- if (metaIn.getLength() > BlockMetadataHeader.getHeaderSize()) {
+ if (!replica.isOnTransientStorage() &&
+ metaIn.getLength() >= BlockMetadataHeader.getHeaderSize()) {
checksumIn = new DataInputStream(new BufferedInputStream(
metaIn, IO_FILE_BUFFER_SIZE));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
index 0e38694..7327420 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
@@ -50,5 +50,7 @@ public class DataNodeFaultInjector {
return false;
}
+ public void stopSendingPacketDownstream() throws IOException {}
+
public void noRegistration() throws IOException { }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
index d7aa79a..0eeb3b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -375,4 +375,57 @@ public class TestClientProtocolForPipelineRecovery {
}
}
}
+
+ /**
+ * Test to make sure the checksum is set correctly after pipeline
+ * recovery transfers 0 byte partial block. If fails the test case
+ * will say "java.io.IOException: Failed to replace a bad datanode
+ * on the existing pipeline due to no more good datanodes being
+ * available to try." This indicates there was a real failure
+ * after the staged failure.
+ */
+ @Test
+ public void testZeroByteBlockRecovery() throws Exception {
+ // Make the first datanode fail once. With 3 nodes and a block being
+ // created with 2 replicas, anything more than this planned failure
+ // will cause a test failure.
+ DataNodeFaultInjector dnFaultInjector = new DataNodeFaultInjector() {
+ int tries = 1;
+ @Override
+ public void stopSendingPacketDownstream() throws IOException {
+ if (tries > 0) {
+ tries--;
+ try {
+ Thread.sleep(60000);
+ } catch (InterruptedException ie) {
+ throw new IOException("Interrupted while sleeping. Bailing out.");
+ }
+ }
+ }
+ };
+ DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.get();
+ DataNodeFaultInjector.set(dnFaultInjector);
+
+ Configuration conf = new HdfsConfiguration();
+ conf.set(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, "1000");
+ conf.set(HdfsClientConfigKeys.
+ BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY, "ALWAYS");
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster.waitActive();
+
+ FileSystem fs = cluster.getFileSystem();
+ FSDataOutputStream out = fs.create(new Path("noheartbeat.dat"), (short)2);
+ out.write(0x31);
+ out.hflush();
+ out.close();
+
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ DataNodeFaultInjector.set(oldDnInjector);
+ }
+ }
}
[21/50] [abbrv] hadoop git commit: HADOOP-12950. ShutdownHookManager
should have a timeout for each of the Registered shutdown hook. Contributed
by Xiaoyu Yao.
Posted by ae...@apache.org.
HADOOP-12950. ShutdownHookManager should have a timeout for each of the Registered shutdown hook. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aac4d65b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aac4d65b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aac4d65b
Branch: refs/heads/HDFS-1312
Commit: aac4d65bf9c6d68f53610e5fe9997a391e3fa053
Parents: 1963978
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Mar 31 15:20:09 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Mar 31 15:22:24 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/util/ShutdownHookManager.java | 116 +++++++++++++++----
.../hadoop/util/TestShutdownHookManager.java | 57 ++++++++-
2 files changed, 150 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aac4d65b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index 843454b..33f942f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -17,8 +17,10 @@
*/
package org.apache.hadoop.util;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
import java.util.ArrayList;
import java.util.Collections;
@@ -26,6 +28,10 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -42,7 +48,12 @@ public class ShutdownHookManager {
private static final ShutdownHookManager MGR = new ShutdownHookManager();
private static final Log LOG = LogFactory.getLog(ShutdownHookManager.class);
+ private static final long TIMEOUT_DEFAULT = 10;
+ private static final TimeUnit TIME_UNIT_DEFAULT = TimeUnit.SECONDS;
+ private static final ExecutorService EXECUTOR =
+ HadoopExecutors.newSingleThreadExecutor(new ThreadFactoryBuilder()
+ .setDaemon(true).build());
static {
try {
Runtime.getRuntime().addShutdownHook(
@@ -50,14 +61,33 @@ public class ShutdownHookManager {
@Override
public void run() {
MGR.shutdownInProgress.set(true);
- for (Runnable hook: MGR.getShutdownHooksInOrder()) {
+ for (HookEntry entry: MGR.getShutdownHooksInOrder()) {
+ Future<?> future = EXECUTOR.submit(entry.getHook());
try {
- hook.run();
+ future.get(entry.getTimeout(), entry.getTimeUnit());
+ } catch (TimeoutException ex) {
+ future.cancel(true);
+ LOG.warn("ShutdownHook '" + entry.getHook().getClass().
+ getSimpleName() + "' timeout, " + ex.toString(), ex);
} catch (Throwable ex) {
- LOG.warn("ShutdownHook '" + hook.getClass().getSimpleName() +
- "' failed, " + ex.toString(), ex);
+ LOG.warn("ShutdownHook '" + entry.getHook().getClass().
+ getSimpleName() + "' failed, " + ex.toString(), ex);
}
}
+ try {
+ EXECUTOR.shutdown();
+ if (!EXECUTOR.awaitTermination(TIMEOUT_DEFAULT,
+ TIME_UNIT_DEFAULT)) {
+ LOG.error("ShutdownHookManger shutdown forcefully.");
+ EXECUTOR.shutdownNow();
+ }
+ LOG.info("ShutdownHookManger complete shutdown.");
+ } catch (InterruptedException ex) {
+ LOG.error("ShutdownHookManger interrupted while waiting for " +
+ "termination.", ex);
+ EXECUTOR.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
}
}
);
@@ -77,15 +107,24 @@ public class ShutdownHookManager {
}
/**
- * Private structure to store ShutdownHook and its priority.
+ * Private structure to store ShutdownHook, its priority and timeout
+ * settings.
*/
- private static class HookEntry {
- Runnable hook;
- int priority;
+ static class HookEntry {
+ private final Runnable hook;
+ private final int priority;
+ private final long timeout;
+ private final TimeUnit unit;
+
+ HookEntry(Runnable hook, int priority) {
+ this(hook, priority, TIMEOUT_DEFAULT, TIME_UNIT_DEFAULT);
+ }
- public HookEntry(Runnable hook, int priority) {
+ HookEntry(Runnable hook, int priority, long timeout, TimeUnit unit) {
this.hook = hook;
this.priority = priority;
+ this.timeout = timeout;
+ this.unit = unit;
}
@Override
@@ -104,10 +143,25 @@ public class ShutdownHookManager {
return eq;
}
+ Runnable getHook() {
+ return hook;
+ }
+
+ int getPriority() {
+ return priority;
+ }
+
+ long getTimeout() {
+ return timeout;
+ }
+
+ TimeUnit getTimeUnit() {
+ return unit;
+ }
}
- private Set<HookEntry> hooks =
- Collections.synchronizedSet(new HashSet<HookEntry>());
+ private final Set<HookEntry> hooks =
+ Collections.synchronizedSet(new HashSet<HookEntry>());
private AtomicBoolean shutdownInProgress = new AtomicBoolean(false);
@@ -121,7 +175,7 @@ public class ShutdownHookManager {
*
* @return the list of shutdownHooks in order of execution.
*/
- List<Runnable> getShutdownHooksInOrder() {
+ List<HookEntry> getShutdownHooksInOrder() {
List<HookEntry> list;
synchronized (MGR.hooks) {
list = new ArrayList<HookEntry>(MGR.hooks);
@@ -134,11 +188,7 @@ public class ShutdownHookManager {
return o2.priority - o1.priority;
}
});
- List<Runnable> ordered = new ArrayList<Runnable>();
- for (HookEntry entry: list) {
- ordered.add(entry.hook);
- }
- return ordered;
+ return list;
}
/**
@@ -154,12 +204,37 @@ public class ShutdownHookManager {
throw new IllegalArgumentException("shutdownHook cannot be NULL");
}
if (shutdownInProgress.get()) {
- throw new IllegalStateException("Shutdown in progress, cannot add a shutdownHook");
+ throw new IllegalStateException("Shutdown in progress, cannot add a " +
+ "shutdownHook");
}
hooks.add(new HookEntry(shutdownHook, priority));
}
/**
+ *
+ * Adds a shutdownHook with a priority and timeout the higher the priority
+ * the earlier will run. ShutdownHooks with same priority run
+ * in a non-deterministic order. The shutdown hook will be terminated if it
+ * has not been finished in the specified period of time.
+ *
+ * @param shutdownHook shutdownHook <code>Runnable</code>
+ * @param priority priority of the shutdownHook
+ * @param timeout timeout of the shutdownHook
+ * @param unit unit of the timeout <code>TimeUnit</code>
+ */
+ public void addShutdownHook(Runnable shutdownHook, int priority, long timeout,
+ TimeUnit unit) {
+ if (shutdownHook == null) {
+ throw new IllegalArgumentException("shutdownHook cannot be NULL");
+ }
+ if (shutdownInProgress.get()) {
+ throw new IllegalStateException("Shutdown in progress, cannot add a " +
+ "shutdownHook");
+ }
+ hooks.add(new HookEntry(shutdownHook, priority, timeout, unit));
+ }
+
+ /**
* Removes a shutdownHook.
*
* @param shutdownHook shutdownHook to remove.
@@ -168,7 +243,8 @@ public class ShutdownHookManager {
*/
public boolean removeShutdownHook(Runnable shutdownHook) {
if (shutdownInProgress.get()) {
- throw new IllegalStateException("Shutdown in progress, cannot remove a shutdownHook");
+ throw new IllegalStateException("Shutdown in progress, cannot remove a " +
+ "shutdownHook");
}
return hooks.remove(new HookEntry(shutdownHook, 0));
}
@@ -198,4 +274,4 @@ public class ShutdownHookManager {
public void clearShutdownHooks() {
hooks.clear();
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aac4d65b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
index 586b899..2aa5e95 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
@@ -17,10 +17,19 @@
*/
package org.apache.hadoop.util;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.LoggerFactory;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.Thread.sleep;
public class TestShutdownHookManager {
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestShutdownHookManager.class.getName());
@Test
public void shutdownHookManager() {
@@ -30,18 +39,48 @@ public class TestShutdownHookManager {
Runnable hook1 = new Runnable() {
@Override
public void run() {
+ LOG.info("Shutdown hook1 complete.");
}
};
Runnable hook2 = new Runnable() {
@Override
public void run() {
+ LOG.info("Shutdown hook2 complete.");
+ }
+ };
+
+ Runnable hook3 = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ sleep(3000);
+ LOG.info("Shutdown hook3 complete.");
+ } catch (InterruptedException ex) {
+ LOG.info("Shutdown hook3 interrupted exception:",
+ ExceptionUtils.getStackTrace(ex));
+ Assert.fail("Hook 3 should not timeout.");
+ }
+ }
+ };
+
+ Runnable hook4 = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ sleep(3500);
+ LOG.info("Shutdown hook4 complete.");
+ Assert.fail("Hook 4 should timeout");
+ } catch (InterruptedException ex) {
+ LOG.info("Shutdown hook4 interrupted exception:",
+ ExceptionUtils.getStackTrace(ex));
+ }
}
};
mgr.addShutdownHook(hook1, 0);
Assert.assertTrue(mgr.hasShutdownHook(hook1));
Assert.assertEquals(1, mgr.getShutdownHooksInOrder().size());
- Assert.assertEquals(hook1, mgr.getShutdownHooksInOrder().get(0));
+ Assert.assertEquals(hook1, mgr.getShutdownHooksInOrder().get(0).getHook());
mgr.removeShutdownHook(hook1);
Assert.assertFalse(mgr.hasShutdownHook(hook1));
@@ -55,8 +94,20 @@ public class TestShutdownHookManager {
Assert.assertTrue(mgr.hasShutdownHook(hook1));
Assert.assertTrue(mgr.hasShutdownHook(hook2));
Assert.assertEquals(2, mgr.getShutdownHooksInOrder().size());
- Assert.assertEquals(hook2, mgr.getShutdownHooksInOrder().get(0));
- Assert.assertEquals(hook1, mgr.getShutdownHooksInOrder().get(1));
+ Assert.assertEquals(hook2, mgr.getShutdownHooksInOrder().get(0).getHook());
+ Assert.assertEquals(hook1, mgr.getShutdownHooksInOrder().get(1).getHook());
+
+ // Test hook finish without timeout
+ mgr.addShutdownHook(hook3, 2, 4, TimeUnit.SECONDS);
+ Assert.assertTrue(mgr.hasShutdownHook(hook3));
+ Assert.assertEquals(hook3, mgr.getShutdownHooksInOrder().get(0).getHook());
+ Assert.assertEquals(4, mgr.getShutdownHooksInOrder().get(0).getTimeout());
+ // Test hook finish with timeout
+ mgr.addShutdownHook(hook4, 3, 2, TimeUnit.SECONDS);
+ Assert.assertTrue(mgr.hasShutdownHook(hook4));
+ Assert.assertEquals(hook4, mgr.getShutdownHooksInOrder().get(0).getHook());
+ Assert.assertEquals(2, mgr.getShutdownHooksInOrder().get(0).getTimeout());
+ LOG.info("Shutdown starts here");
}
}
[08/50] [abbrv] hadoop git commit: HDFS-10221. Add .json to the rat
exclusions. Contributed by Ming Ma.
Posted by ae...@apache.org.
HDFS-10221. Add .json to the rat exclusions. Contributed by Ming Ma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/32c0c3ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/32c0c3ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/32c0c3ec
Branch: refs/heads/HDFS-1312
Commit: 32c0c3ecdf72e89a63f4aee5e75d1c5a12714b89
Parents: e4fc609
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Mar 31 09:04:09 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Mar 31 09:06:14 2016 +0900
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32c0c3ec/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 4e1901b5..668bbfe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -381,6 +381,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>src/test/all-tests</exclude>
<exclude>src/test/resources/*.tgz</exclude>
<exclude>src/test/resources/data*</exclude>
+ <exclude>**/*.json</exclude>
<exclude>src/test/resources/editsStored*</exclude>
<exclude>src/test/resources/empty-file</exclude>
<exclude>src/main/webapps/datanode/robots.txt</exclude>
[10/50] [abbrv] hadoop git commit: YARN-4884. Fix missing
documentation about rmadmin command regarding node labels. Contributed by Kai
Sasaki.
Posted by ae...@apache.org.
YARN-4884. Fix missing documentation about rmadmin command regarding node labels. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1b8f6b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1b8f6b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1b8f6b2
Branch: refs/heads/HDFS-1312
Commit: f1b8f6b2c16403869f78a54268ae1165982a7050
Parents: acca149
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Mar 31 14:01:48 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Mar 31 14:01:48 2016 +0530
----------------------------------------------------------------------
.../src/site/markdown/YarnCommands.md | 45 +++++++++++++-------
1 file changed, 30 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b8f6b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 5941988..40704f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -217,31 +217,46 @@ Start the ResourceManager
Usage:
```
- yarn rmadmin [-refreshQueues]
- [-refreshNodes]
- [-refreshUserToGroupsMapping]
- [-refreshSuperUserGroupsConfiguration]
- [-refreshAdminAcls]
- [-refreshServiceAcl]
- [-getGroups [username]]
- [-transitionToActive [--forceactive] [--forcemanual] <serviceId>]
- [-transitionToStandby [--forcemanual] <serviceId>]
- [-failover [--forcefence] [--forceactive] <serviceId1> <serviceId2>]
- [-getServiceState <serviceId>]
- [-checkHealth <serviceId>]
- [-help [cmd]]
+ Usage: yarn rmadmin
+ -refreshQueues
+ -refreshNodes [-g [timeout in seconds]]
+ -refreshNodesResources
+ -refreshSuperUserGroupsConfiguration
+ -refreshUserToGroupsMappings
+ -refreshAdminAcls
+ -refreshServiceAcl
+ -getGroups [username]
+ -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3">
+ -removeFromClusterNodeLabels <label1,label2,label3> (label splitted by ",")
+ -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2">
+ -directlyAccessNodeLabelStore
+ -refreshClusterMaxPriority
+ -updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout])
+ -transitionToActive [--forceactive] <serviceId>
+ -transitionToStandby <serviceId>
+ -failover [--forcefence] [--forceactive] <serviceId> <serviceId>
+ -getServiceState <serviceId>
+ -checkHealth <serviceId>
+ -help [cmd]
```
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| -refreshQueues | Reload the queues' acls, states and scheduler specific properties. ResourceManager will reload the mapred-queues configuration file. |
| -refreshNodes | Refresh the hosts information at the ResourceManager. |
-| -refreshUserToGroupsMappings | Refresh user-to-groups mappings. |
+| -refreshNodesResources | Refresh resources of NodeManagers at the ResourceManager. |
| -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings. |
+| -refreshUserToGroupsMappings | Refresh user-to-groups mappings. |
| -refreshAdminAcls | Refresh acls for administration of ResourceManager |
| -refreshServiceAcl | Reload the service-level authorization policy file ResourceManager will reload the authorization policy file. |
| -getGroups [username] | Get groups the specified user belongs to. |
-| -transitionToActive [--forceactive] [--forcemanual] \<serviceId\> | Transitions the service into Active state. Try to make the target active without checking that there is no active node if the --forceactive option is used. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. |
+| -addToClusterNodeLabels <"label1(exclusive=true),label2(exclusive=false),label3"> | Add to cluster node labels. Default exclusivity is true. |
+| -removeFromClusterNodeLabels <label1,label2,label3> (label splitted by ",") | Remove from cluster node labels. |
+| -replaceLabelsOnNode <"node1[:port]=label1,label2 node2[:port]=label1,label2"> | Replace labels on nodes (please note that we do not support specifying multiple labels on a single host for now.) |
+| -directlyAccessNodeLabelStore | This is DEPRECATED, will be removed in future releases. Directly access node label store, with this option, all node label related operations will not connect RM. Instead, they will access/modify stored node labels directly. By default, it is false (access via RM). AND PLEASE NOTE: if you configured yarn.node-labels.fs-store.root-dir to a local directory (instead of NFS or HDFS), this option will only work when the command run on the machine where RM is running. |
+| -refreshClusterMaxPriority | Refresh cluster max priority |
+| -updateNodeResource [NodeID] [MemSize] [vCores] \([OvercommitTimeout]\) | Update resource on specific node. |
+| -transitionToActive [--forceactive] [--forcemanual] \<serviceId\> | Transitions the service into Active state. Try to make the target active without checking that there is no active node if the --forceactive option is used. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. This command can not be used if automatic failover is enabled.|
| -transitionToStandby [--forcemanual] \<serviceId\> | Transitions the service into Standby state. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. |
| -failover [--forceactive] \<serviceId1\> \<serviceId2\> | Initiate a failover from serviceId1 to serviceId2. Try to failover to the target service even if it is not ready if the --forceactive option is used. This command can not be used if automatic failover is enabled. |
| -getServiceState \<serviceId\> | Returns the state of the service. |
[22/50] [abbrv] hadoop git commit: HADOOP-11661. Deprecate
FileUtil#copyMerge. Contributed by Brahma Reddy Battula.
Posted by ae...@apache.org.
HADOOP-11661. Deprecate FileUtil#copyMerge. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8d8b80a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8d8b80a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8d8b80a
Branch: refs/heads/HDFS-1312
Commit: a8d8b80a205c78bf0aa65d32a6a83c16d3ea3f0b
Parents: aac4d65
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Apr 1 13:59:14 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Apr 1 13:59:14 2016 +0900
----------------------------------------------------------------------
.../hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java | 1 +
.../src/test/java/org/apache/hadoop/fs/TestFileUtil.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8d8b80a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index e74c41c..b855c48 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -381,6 +381,7 @@ public class FileUtil {
}
+ @Deprecated
/** Copy all files in a directory to one output file (merge). */
public static boolean copyMerge(FileSystem srcFS, Path srcDir,
FileSystem dstFS, Path dstFile,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8d8b80a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 5fc0b2d..f7464b7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -559,6 +559,7 @@ public class TestFileUtil {
* @return boolean true if the call to FileUtil.copyMerge was successful.
* @throws IOException if an I/O error occurs.
*/
+ @SuppressWarnings("deprecation")
private boolean copyMerge(String src, String dst)
throws IOException {
Configuration conf = new Configuration();
[12/50] [abbrv] hadoop git commit: HADOOP-11393. Revert HADOOP_PREFIX,
go back to HADOOP_HOME (aw)
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
index 6470393..a7a001c 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
@@ -89,7 +89,7 @@ public class LoadTypedBytes implements Tool {
}
private void printUsage() {
- System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar"
+ System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
+ " loadtb <path>");
System.out.println(" Reads typed bytes from standard input" +
" and stores them in a sequence file in");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
index 118e0fb..9f5b293 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
@@ -502,7 +502,7 @@ public class StreamJob implements Tool {
}
private void printUsage(boolean detailed) {
- System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar"
+ System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
+ " [options]");
System.out.println("Options:");
System.out.println(" -input <path> DFS input file(s) for the Map"
@@ -551,7 +551,7 @@ public class StreamJob implements Tool {
System.out.println();
System.out.println("For more details about these options:");
System.out.println("Use " +
- "$HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar -info");
+ "$HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar -info");
return;
}
System.out.println();
@@ -611,7 +611,7 @@ public class StreamJob implements Tool {
System.out.println(" -D stream.non.zero.exit.is.failure=false");
System.out.println("Use a custom hadoop streaming build along with standard"
+ " hadoop install:");
- System.out.println(" $HADOOP_PREFIX/bin/hadoop jar " +
+ System.out.println(" $HADOOP_HOME/bin/hadoop jar " +
"/path/my-hadoop-streaming.jar [...]\\");
System.out.println(" [...] -D stream.shipped.hadoopstreaming=" +
"/path/my-hadoop-streaming.jar");
@@ -625,7 +625,7 @@ public class StreamJob implements Tool {
System.out.println(" -cmdenv EXAMPLE_DIR=/home/example/dictionaries/");
System.out.println();
System.out.println("Shortcut:");
- System.out.println(" setenv HSTREAMING \"$HADOOP_PREFIX/bin/hadoop jar " +
+ System.out.println(" setenv HSTREAMING \"$HADOOP_HOME/bin/hadoop jar " +
"hadoop-streaming.jar\"");
System.out.println();
System.out.println("Example: $HSTREAMING -mapper " +
@@ -648,9 +648,9 @@ public class StreamJob implements Tool {
// --------------------------------------------
protected String getHadoopClientHome() {
- String h = env_.getProperty("HADOOP_PREFIX"); // standard Hadoop
+ String h = env_.getProperty("HADOOP_HOME"); // standard Hadoop
if (h == null) {
- //fail("Missing required environment variable: HADOOP_PREFIX");
+ //fail("Missing required environment variable: HADOOP_HOME");
h = "UNDEF";
}
return h;
@@ -674,8 +674,8 @@ public class StreamJob implements Tool {
// usually found in: build/contrib or build/hadoop-<version>-dev-streaming.jar
// First try an explicit spec: it's too hard to find our own location in this case:
- // $HADOOP_PREFIX/bin/hadoop jar /not/first/on/classpath/custom-hadoop-streaming.jar
- // where findInClasspath() would find the version of hadoop-streaming.jar in $HADOOP_PREFIX
+ // $HADOOP_HOME/bin/hadoop jar /not/first/on/classpath/custom-hadoop-streaming.jar
+ // where findInClasspath() would find the version of hadoop-streaming.jar in $HADOOP_HOME
String runtimeClasses = config_.get("stream.shipped.hadoopstreaming"); // jar or class dir
if (runtimeClasses == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
index f462fad..3b41299 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
@@ -26,8 +26,8 @@ function hadoop_usage
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
index 3305989..358f0c9 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
@@ -26,8 +26,8 @@ function hadoop_usage
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index cb2364b..cac3bb6 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -51,8 +51,8 @@ function hadoop_usage
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
index 7df9fa1..d7fa406 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
@@ -53,7 +53,7 @@ function hadoop_subproject_init
hadoop_deprecate_envvar YARN_SLAVES HADOOP_SLAVES
- HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}"
+ HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_HOME}"
# YARN-1429 added the completely superfluous YARN_USER_CLASSPATH
# env var. We're going to override HADOOP_USER_CLASSPATH to keep
@@ -74,8 +74,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then
- . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
+elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
else
echo "ERROR: Hadoop common not found." 2>&1
exit 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
index 2f886f2..a195c60 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
@@ -21,8 +21,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
index 4ef0864..958c8bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
@@ -25,8 +25,8 @@ this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
index c4500c4..27ff438 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
@@ -169,7 +169,7 @@ public class TestDockerContainerExecutorWithMocks {
String appSubmitter = "nobody";
String appId = "APP_ID";
String containerId = "CONTAINER_ID";
- String testImage = "testrepo.com/test-image rm -rf $HADOOP_PREFIX/*";
+ String testImage = "testrepo.com/test-image rm -rf $HADOOP_HOME/*";
Container container = mock(Container.class, RETURNS_DEEP_STUBS);
ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
index fbfe04b..6a7db43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm
@@ -142,7 +142,7 @@ Step 2. Pick a custom Docker image if you want. In this example, we'll use seque
Step 3. Run.
```bash
-hadoop jar $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar \
+hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar \
teragen \
-Dmapreduce.map.env="yarn.nodemanager.docker-container-executor.image-name=sequenceiq/hadoop-docker:2.4.1" \
-Dyarn.app.mapreduce.am.env="yarn.nodemanager.docker-container-executor.image-name=sequenceiq/hadoop-docker:2.4.1" \
[42/50] [abbrv] hadoop git commit: YARN-4609. RM Nodes list page
takes too much time to load. Contributed by Bibin A Chundatt
Posted by ae...@apache.org.
YARN-4609. RM Nodes list page takes too much time to load. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/776b549e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/776b549e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/776b549e
Branch: refs/heads/HDFS-1312
Commit: 776b549e2ac20a68a5513cbcaac0edc33233dc03
Parents: 552237d
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Apr 5 14:47:25 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Apr 5 14:47:25 2016 +0530
----------------------------------------------------------------------
.../resourcemanager/webapp/NodesPage.java | 53 +++++++++++++-------
.../resourcemanager/webapp/TestNodesPage.java | 37 ++++----------
2 files changed, 45 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/776b549e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 9603468..7063421 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
@@ -101,6 +100,7 @@ class NodesPage extends RmView {
LOG.debug("Unexpected state filter for inactive RM node");
}
}
+ StringBuilder nodeTableData = new StringBuilder("[\n");
for (RMNode ni : rmNodes) {
if (stateFilter != null) {
NodeState state = ni.getState();
@@ -129,27 +129,40 @@ class NodesPage extends RmView {
NodeInfo info = new NodeInfo(ni, sched);
int usedMemory = (int) info.getUsedMemory();
int availableMemory = (int) info.getAvailableMemory();
- TR<TBODY<TABLE<Hamlet>>> row =
- tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
- .td(info.getRack()).td(info.getState()).td(info.getNodeId());
+ nodeTableData.append("[\"")
+ .append(StringUtils.join(",", info.getNodeLabels())).append("\",\"")
+ .append(info.getRack()).append("\",\"").append(info.getState())
+ .append("\",\"").append(info.getNodeId());
if (isInactive) {
- row.td()._("N/A")._();
+ nodeTableData.append("\",\"").append("N/A").append("\",\"");
} else {
String httpAddress = info.getNodeHTTPAddress();
- row.td().a("//" + httpAddress, httpAddress)._();
+ nodeTableData.append("\",\"<a ").append("href='" + "//" + httpAddress)
+ .append("'>").append(httpAddress).append("</a>\",").append("\"");
}
- row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
- ._(Times.format(info.getLastHealthUpdate()))._()
- .td(info.getHealthReport())
- .td(String.valueOf(info.getNumContainers())).td().br()
- .$title(String.valueOf(usedMemory))._()
- ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
- .$title(String.valueOf(availableMemory))._()
- ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
- .td(String.valueOf(info.getUsedVirtualCores()))
- .td(String.valueOf(info.getAvailableVirtualCores()))
- .td(ni.getNodeManagerVersion())._();
+ nodeTableData.append("<br title='")
+ .append(String.valueOf(info.getLastHealthUpdate())).append("'>")
+ .append(Times.format(info.getLastHealthUpdate())).append("\",\"")
+ .append(info.getHealthReport()).append("\",\"")
+ .append(String.valueOf(info.getNumContainers())).append("\",\"")
+ .append("<br title='").append(String.valueOf(usedMemory))
+ .append("'>").append(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))
+ .append("\",\"").append("<br title='")
+ .append(String.valueOf(availableMemory)).append("'>")
+ .append(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))
+ .append("\",\"").append(String.valueOf(info.getUsedVirtualCores()))
+ .append("\",\"")
+ .append(String.valueOf(info.getAvailableVirtualCores()))
+ .append("\",\"").append(ni.getNodeManagerVersion())
+ .append("\"],\n");
+ }
+ if (nodeTableData.charAt(nodeTableData.length() - 2) == ',') {
+ nodeTableData.delete(nodeTableData.length() - 2,
+ nodeTableData.length() - 1);
}
+ nodeTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var nodeTableData=" + nodeTableData)._();
tbody._()._();
}
}
@@ -175,7 +188,9 @@ class NodesPage extends RmView {
}
private String nodesTableInit() {
- StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ StringBuilder b = tableInit().append(", 'aaData': nodeTableData")
+ .append(", bDeferRender: true").append(", bProcessing: true")
+ .append(", aoColumnDefs: [");
b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+ "'aTargets': [ 8, 9 ] }");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/776b549e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index 0699c0e..71d077b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -81,11 +81,8 @@ public class TestNodesPage {
Mockito.verify(writer,
Mockito.times(numberOfActualTableHeaders + numberOfThInMetricsTable))
.print("<th");
- Mockito.verify(
- writer,
- Mockito.times(numberOfRacks * numberOfNodesPerRack
- * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
- "<td");
+ Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+ .print("<td");
}
@Test
@@ -99,11 +96,8 @@ public class TestNodesPage {
Mockito.verify(writer,
Mockito.times(numberOfActualTableHeaders + numberOfThInMetricsTable))
.print("<th");
- Mockito.verify(
- writer,
- Mockito.times(numberOfRacks * numberOfLostNodesPerRack
- * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
- "<td");
+ Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+ .print("<td");
}
@Test
@@ -113,12 +107,9 @@ public class TestNodesPage {
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
-
- Mockito.verify(
- writer,
- Mockito.times(numberOfRacks
- * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
- "<td");
+ Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+ .print("<td");
+ Mockito.verify(writer, Mockito.times(1)).print("<script");
}
@Test
@@ -129,11 +120,8 @@ public class TestNodesPage {
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
- Mockito.verify(
- writer,
- Mockito.times(numberOfRacks * (numberOfNodesPerRack - 1)
- * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
- "<td");
+ Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+ .print("<td");
}
@Test
@@ -144,10 +132,7 @@ public class TestNodesPage {
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
- Mockito.verify(
- writer,
- Mockito.times(numberOfRacks * numberOfNodesPerRack
- * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
- "<td");
+ Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
+ .print("<td");
}
}
[25/50] [abbrv] hadoop git commit: YARN-4895. Add subtractFrom method
to ResourceUtilization class. Contributed by Konstantinos Karanasos.
Posted by ae...@apache.org.
YARN-4895. Add subtractFrom method to ResourceUtilization class. Contributed by Konstantinos Karanasos.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82621e38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82621e38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82621e38
Branch: refs/heads/HDFS-1312
Commit: 82621e38a0445832998bc00693279e23a98605c1
Parents: 256c82f
Author: Arun Suresh <as...@apache.org>
Authored: Fri Apr 1 14:57:06 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Apr 1 14:57:06 2016 -0700
----------------------------------------------------------------------
.../yarn/api/records/ResourceUtilization.java | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82621e38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
index 5f52f85..2ae4872 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
@@ -44,6 +44,14 @@ public abstract class ResourceUtilization implements
return utilization;
}
+ @Public
+ @Unstable
+ public static ResourceUtilization newInstance(
+ ResourceUtilization resourceUtil) {
+ return newInstance(resourceUtil.getPhysicalMemory(),
+ resourceUtil.getVirtualMemory(), resourceUtil.getCPU());
+ }
+
/**
* Get used <em>virtual memory</em>.
*
@@ -147,4 +155,18 @@ public abstract class ResourceUtilization implements
this.setVirtualMemory(this.getVirtualMemory() + vmem);
this.setCPU(this.getCPU() + cpu);
}
+
+ /**
+ * Subtract utilization from the current one.
+ * @param pmem Physical memory to be subtracted.
+ * @param vmem Virtual memory to be subtracted.
+ * @param cpu CPU utilization to be subtracted.
+ */
+ @Public
+ @Unstable
+ public void subtractFrom(int pmem, int vmem, float cpu) {
+ this.setPhysicalMemory(this.getPhysicalMemory() - pmem);
+ this.setVirtualMemory(this.getVirtualMemory() - vmem);
+ this.setCPU(this.getCPU() - cpu);
+ }
}
\ No newline at end of file
[29/50] [abbrv] hadoop git commit: HADOOP-12169 ListStatus on empty
dir in S3A lists itself instead of returning an empty list. author: Pieter
Reuse.
Posted by ae...@apache.org.
HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ecdd4cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ecdd4cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ecdd4cf
Branch: refs/heads/HDFS-1312
Commit: 0ecdd4cffa51e34997321c384496efc249e3d8ff
Parents: 54b2e78
Author: Steve Loughran <st...@apache.org>
Authored: Sun Apr 3 16:39:14 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Apr 3 16:40:19 2016 +0100
----------------------------------------------------------------------
.../AbstractContractGetFileStatusTest.java | 23 ++++++++++++++++++++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 7 ++++--
.../src/test/resources/contract/s3a.xml | 5 +++++
3 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecdd4cf/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
index 7ed375e..3e5bb12 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
@@ -19,10 +19,14 @@
package org.apache.hadoop.fs.contract;
import java.io.FileNotFoundException;
+import java.io.IOException;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,4 +62,23 @@ public abstract class AbstractContractGetFileStatusTest extends
handleExpectedException(e);
}
}
+
+ @Test
+ public void testListStatusEmptyDirectory() throws IOException {
+ // remove the test directory
+ FileSystem fs = getFileSystem();
+ assertTrue(fs.delete(getContract().getTestPath(), true));
+
+ // create a - non-qualified - Path for a subdir
+ Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName());
+ assertTrue(fs.mkdirs(subfolder));
+
+ // assert empty ls on the empty dir
+ assertEquals("ls on an empty directory not of length 0", 0,
+ fs.listStatus(subfolder).length);
+
+ // assert non-empty ls on parent dir
+ assertTrue("ls on a non-empty directory of length 0",
+ fs.listStatus(getContract().getTestPath()).length > 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecdd4cf/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 6afb05d..fe705ce 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -788,11 +788,14 @@ public class S3AFileSystem extends FileSystem {
ObjectListing objects = s3.listObjects(request);
statistics.incrementReadOps(1);
+ Path fQualified = f.makeQualified(uri, workingDir);
+
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
// Skip over keys that are ourselves and old S3N _$folder$ files
- if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
+ if (keyPath.equals(fQualified) ||
+ summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignoring: " + keyPath);
}
@@ -807,7 +810,7 @@ public class S3AFileSystem extends FileSystem {
} else {
result.add(new S3AFileStatus(summary.getSize(),
dateToLong(summary.getLastModified()), keyPath,
- getDefaultBlockSize(f.makeQualified(uri, workingDir))));
+ getDefaultBlockSize(fQualified)));
if (LOG.isDebugEnabled()) {
LOG.debug("Adding: fi: " + keyPath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecdd4cf/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
index 4f9c081..be1e7ca 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
@@ -78,6 +78,11 @@
</property>
<property>
+ <name>fs.contract.supports-getfilestatus</name>
+ <value>true</value>
+ </property>
+
+ <property>
<name>fs.contract.supports-seek</name>
<value>true</value>
</property>
[18/50] [abbrv] hadoop git commit: YARN-4811. Generate histograms in
ContainerMetrics for actual container resource usage
Posted by ae...@apache.org.
YARN-4811. Generate histograms in ContainerMetrics for actual container resource usage
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0dd9bcab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0dd9bcab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0dd9bcab
Branch: refs/heads/HDFS-1312
Commit: 0dd9bcab97ccdf24a2174636604110b74664cf80
Parents: 7a02147
Author: Jian He <ji...@apache.org>
Authored: Thu Mar 31 14:28:13 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Mar 31 14:28:13 2016 -0700
----------------------------------------------------------------------
.../hadoop/metrics2/lib/MutableQuantiles.java | 7 +-
.../hadoop/metrics2/util/QuantileEstimator.java | 32 +++++++++
.../hadoop/metrics2/util/SampleQuantiles.java | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 5 ++
.../monitor/ContainerMetrics.java | 69 ++++++++++++++++++++
.../monitor/TestContainerMetrics.java | 58 +++++++++++++++-
6 files changed, 170 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dd9bcab/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
index 2e6053f..a4711db 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.util.Quantile;
+import org.apache.hadoop.metrics2.util.QuantileEstimator;
import org.apache.hadoop.metrics2.util.SampleQuantiles;
import com.google.common.annotations.VisibleForTesting;
@@ -54,7 +55,7 @@ public class MutableQuantiles extends MutableMetric {
private final MetricsInfo[] quantileInfos;
private final int interval;
- private SampleQuantiles estimator;
+ private QuantileEstimator estimator;
private long previousCount = 0;
@VisibleForTesting
@@ -134,6 +135,10 @@ public class MutableQuantiles extends MutableMetric {
return interval;
}
+ public synchronized void setEstimator(QuantileEstimator quantileEstimator) {
+ this.estimator = quantileEstimator;
+ }
+
/**
* Runnable used to periodically roll over the internal
* {@link SampleQuantiles} every interval.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dd9bcab/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java
new file mode 100644
index 0000000..075b879
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/QuantileEstimator.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.util;
+
+import java.util.Map;
+
+public interface QuantileEstimator {
+
+ void insert(long value);
+
+ Map<Quantile, Long> snapshot();
+
+ long getCount();
+
+ void clear();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dd9bcab/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
index d540036..0c5d98f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
@@ -47,7 +47,7 @@ import com.google.common.base.Preconditions;
*
*/
@InterfaceAudience.Private
-public class SampleQuantiles {
+public class SampleQuantiles implements QuantileEstimator {
/**
* Total number of items in stream
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dd9bcab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 051bb4e..59c3332 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -117,6 +117,11 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
<!--
junit must be before mockito-all on the classpath. mockito-all bundles its
own copy of the hamcrest classes, but they don't match our junit version.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dd9bcab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
index 48128c1..9d17db0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.UniformReservoir;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -29,13 +32,17 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableStat;
+import org.apache.hadoop.metrics2.util.Quantile;
+import org.apache.hadoop.metrics2.util.QuantileEstimator;
import org.apache.hadoop.yarn.api.records.ContainerId;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.TreeMap;
import static org.apache.hadoop.metrics2.lib.Interns.info;
@@ -47,10 +54,13 @@ public class ContainerMetrics implements MetricsSource {
public static final String VMEM_LIMIT_METRIC_NAME = "vMemLimitMBs";
public static final String VCORE_LIMIT_METRIC_NAME = "vCoreLimit";
public static final String PMEM_USAGE_METRIC_NAME = "pMemUsageMBs";
+ public static final String PMEM_USAGE_QUANTILES_NAME = "pMemUsageMBHistogram";
public static final String LAUNCH_DURATION_METRIC_NAME = "launchDurationMs";
public static final String LOCALIZATION_DURATION_METRIC_NAME =
"localizationDurationMs";
private static final String PHY_CPU_USAGE_METRIC_NAME = "pCpuUsagePercent";
+ private static final String PHY_CPU_USAGE_QUANTILES_NAME =
+ "pCpuUsagePercentHistogram";
// Use a multiplier of 1000 to avoid losing too much precision when
// converting to integers
@@ -59,6 +69,9 @@ public class ContainerMetrics implements MetricsSource {
@Metric
public MutableStat pMemMBsStat;
+ @Metric
+ public MutableQuantiles pMemMBQuantiles;
+
// This tracks overall CPU percentage of the machine in terms of percentage
// of 1 core similar to top
// Thus if you use 2 cores completely out of 4 available cores this value
@@ -67,6 +80,9 @@ public class ContainerMetrics implements MetricsSource {
public MutableStat cpuCoreUsagePercent;
@Metric
+ public MutableQuantiles cpuCoreUsagePercentQuantiles;
+
+ @Metric
public MutableStat milliVcoresUsed;
@Metric
@@ -127,9 +143,23 @@ public class ContainerMetrics implements MetricsSource {
this.pMemMBsStat = registry.newStat(
PMEM_USAGE_METRIC_NAME, "Physical memory stats", "Usage", "MBs", true);
+ this.pMemMBQuantiles = registry
+ .newQuantiles(PMEM_USAGE_QUANTILES_NAME, "Physical memory quantiles",
+ "Usage", "MBs", 1);
+ ContainerMetricsQuantiles memEstimator =
+ new ContainerMetricsQuantiles(MutableQuantiles.quantiles);
+ pMemMBQuantiles.setEstimator(memEstimator);
+
this.cpuCoreUsagePercent = registry.newStat(
PHY_CPU_USAGE_METRIC_NAME, "Physical Cpu core percent usage stats",
"Usage", "Percents", true);
+ this.cpuCoreUsagePercentQuantiles = registry
+ .newQuantiles(PHY_CPU_USAGE_QUANTILES_NAME,
+ "Physical Cpu core percent usage quantiles", "Usage", "Percents",
+ 1);
+ ContainerMetricsQuantiles cpuEstimator =
+ new ContainerMetricsQuantiles(MutableQuantiles.quantiles);
+ cpuCoreUsagePercentQuantiles.setEstimator(cpuEstimator);
this.milliVcoresUsed = registry.newStat(
VCORE_USAGE_METRIC_NAME, "1000 times Vcore usage", "Usage",
"MilliVcores", true);
@@ -216,6 +246,7 @@ public class ContainerMetrics implements MetricsSource {
public void recordMemoryUsage(int memoryMBs) {
if (memoryMBs >= 0) {
this.pMemMBsStat.add(memoryMBs);
+ this.pMemMBQuantiles.add(memoryMBs);
}
}
@@ -223,6 +254,7 @@ public class ContainerMetrics implements MetricsSource {
int totalPhysicalCpuPercent, int milliVcoresUsed) {
if (totalPhysicalCpuPercent >=0) {
this.cpuCoreUsagePercent.add(totalPhysicalCpuPercent);
+ this.cpuCoreUsagePercentQuantiles.add(totalPhysicalCpuPercent);
}
if (milliVcoresUsed >= 0) {
this.milliVcoresUsed.add(milliVcoresUsed);
@@ -274,4 +306,41 @@ public class ContainerMetrics implements MetricsSource {
};
unregisterContainerMetricsTimer.schedule(timerTask, unregisterDelayMs);
}
+
+ public static class ContainerMetricsQuantiles implements QuantileEstimator {
+
+ private final Histogram histogram = new Histogram(new UniformReservoir());
+
+ private Quantile[] quantiles;
+
+ ContainerMetricsQuantiles(Quantile[] q) {
+ quantiles = q;
+ }
+
+ @Override
+ public synchronized void insert(long value) {
+ histogram.update(value);
+ }
+
+ @Override
+ synchronized public long getCount() {
+ return histogram.getCount();
+ }
+
+ @Override
+ synchronized public void clear() {
+ // don't do anything because we want metrics over the lifetime of the
+ // container
+ }
+
+ @Override
+ public synchronized Map<Quantile, Long> snapshot() {
+ Snapshot snapshot = histogram.getSnapshot();
+ Map<Quantile, Long> values = new TreeMap<>();
+ for (Quantile quantile : quantiles) {
+ values.put(quantile, (long) snapshot.getValue(quantile.quantile));
+ }
+ return values;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0dd9bcab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
index 2beb927..fb482c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
@@ -26,10 +27,15 @@ import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.junit.Assert;
import org.junit.Test;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
@@ -158,4 +164,54 @@ public class TestContainerMetrics {
system, containerId3, 1, 0));
system.shutdown();
}
+
+ /**
+ * Run a test to submit values for actual memory usage and see if the
+ * histogram comes out correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testContainerMetricsHistogram() throws Exception {
+
+ // submit 2 values - 1024 and 2048. 75th, 90th, 95th and 99th percentiles
+ // will be 2048. 50th percentile will be 1536((1024+2048)/2)
+ // if we keep recording 1024 and 2048 in a loop, the 50th percentile
+ // will tend closer to 2048
+ Map<String, Long> expectedValues = new HashMap<>();
+ expectedValues.put("PMemUsageMBHistogram50thPercentileMBs", 1536L);
+ expectedValues.put("PMemUsageMBHistogram75thPercentileMBs", 2048L);
+ expectedValues.put("PMemUsageMBHistogram90thPercentileMBs", 2048L);
+ expectedValues.put("PMemUsageMBHistogram95thPercentileMBs", 2048L);
+ expectedValues.put("PMemUsageMBHistogram99thPercentileMBs", 2048L);
+ expectedValues.put("PCpuUsagePercentHistogram50thPercentilePercents", 0L);
+ expectedValues.put("PCpuUsagePercentHistogram75thPercentilePercents", 0L);
+ expectedValues.put("PCpuUsagePercentHistogram90thPercentilePercents", 0L);
+ expectedValues.put("PCpuUsagePercentHistogram95thPercentilePercents", 0L);
+ expectedValues.put("PCpuUsagePercentHistogram99thPercentilePercents", 0L);
+ Set<String> testResults = new HashSet<>();
+ int delay = 10;
+ int rolloverDelay = 1000;
+ MetricsCollectorImpl collector = new MetricsCollectorImpl();
+ ContainerId containerId = mock(ContainerId.class);
+ ContainerMetrics metrics =
+ ContainerMetrics.forContainer(containerId, delay, 0);
+
+ metrics.recordMemoryUsage(1024);
+ metrics.recordMemoryUsage(2048);
+ Thread.sleep(rolloverDelay + 10);
+ metrics.getMetrics(collector, true);
+ for (MetricsRecord record : collector.getRecords()) {
+ for (AbstractMetric metric : record.metrics()) {
+ String metricName = metric.name();
+ if (expectedValues.containsKey(metricName)) {
+ Long expectedValue = expectedValues.get(metricName);
+ Assert.assertEquals(
+ "Metric " + metricName + " doesn't have expected value",
+ expectedValue, metric.value());
+ testResults.add(metricName);
+ }
+ }
+ }
+ Assert.assertEquals(expectedValues.keySet(), testResults);
+ }
}
[04/50] [abbrv] hadoop git commit: YARN-4822. Refactor existing
Preemption Policy of CS for easier adding new approach to select preemption
candidates. Contributed by Wangda Tan
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 9b499c8..7e668b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -17,26 +17,13 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.TreeSet;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -50,7 +37,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.util.Clock;
@@ -58,8 +44,16 @@ import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* This class implement a {@link SchedulingEditPolicy} that is designed to be
@@ -80,79 +74,59 @@ import com.google.common.collect.ImmutableSet;
* this policy will trigger forced termination of containers (again by generating
* {@link ContainerPreemptEvent}).
*/
-public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolicy {
-
+public class ProportionalCapacityPreemptionPolicy
+ implements SchedulingEditPolicy, CapacitySchedulerPreemptionContext {
private static final Log LOG =
LogFactory.getLog(ProportionalCapacityPreemptionPolicy.class);
- /** If true, run the policy but do not affect the cluster with preemption and
- * kill events. */
- public static final String OBSERVE_ONLY =
- "yarn.resourcemanager.monitor.capacity.preemption.observe_only";
- /** Time in milliseconds between invocations of this policy */
- public static final String MONITORING_INTERVAL =
- "yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval";
- /** Time in milliseconds between requesting a preemption from an application
- * and killing the container. */
- public static final String WAIT_TIME_BEFORE_KILL =
- "yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill";
- /** Maximum percentage of resources preempted in a single round. By
- * controlling this value one can throttle the pace at which containers are
- * reclaimed from the cluster. After computing the total desired preemption,
- * the policy scales it back within this limit. */
- public static final String TOTAL_PREEMPTION_PER_ROUND =
- "yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round";
- /** Maximum amount of resources above the target capacity ignored for
- * preemption. This defines a deadzone around the target capacity that helps
- * prevent thrashing and oscillations around the computed target balance.
- * High values would slow the time to capacity and (absent natural
- * completions) it might prevent convergence to guaranteed capacity. */
- public static final String MAX_IGNORED_OVER_CAPACITY =
- "yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity";
- /**
- * Given a computed preemption target, account for containers naturally
- * expiring and preempt only this percentage of the delta. This determines
- * the rate of geometric convergence into the deadzone ({@link
- * #MAX_IGNORED_OVER_CAPACITY}). For example, a termination factor of 0.5
- * will reclaim almost 95% of resources within 5 * {@link
- * #WAIT_TIME_BEFORE_KILL}, even absent natural termination. */
- public static final String NATURAL_TERMINATION_FACTOR =
- "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor";
-
- private RMContext rmContext;
-
private final Clock clock;
+
+ // Configurable fields
private double maxIgnoredOverCapacity;
private long maxWaitTime;
- private CapacityScheduler scheduler;
private long monitoringInterval;
- private final Map<RMContainer, Long> preempted = new HashMap<>();
-
- private ResourceCalculator rc;
private float percentageClusterPreemptionAllowed;
private double naturalTerminationFactor;
private boolean observeOnly;
+ private boolean lazyPreempionEnabled;
+
+ // Pointer to other RM components
+ private RMContext rmContext;
+ private ResourceCalculator rc;
+ private CapacityScheduler scheduler;
+ private RMNodeLabelsManager nlm;
+
+ // Internal properties to make decisions of what to preempt
+ private final Map<RMContainer,Long> preemptionCandidates =
+ new HashMap<>();
private Map<String, Map<String, TempQueuePerPartition>> queueToPartitions =
new HashMap<>();
- private RMNodeLabelsManager nlm;
+ private List<PreemptionCandidatesSelector>
+ candidatesSelectionPolicies = new ArrayList<>();
+ private Set<String> allPartitions;
+ private Set<String> leafQueueNames;
// Preemptable Entities, synced from scheduler at every run
- private Map<String, PreemptableQueue> preemptableEntities = null;
+ private Map<String, PreemptableQueue> preemptableQueues;
private Set<ContainerId> killableContainers;
+ @SuppressWarnings("unchecked")
public ProportionalCapacityPreemptionPolicy() {
clock = SystemClock.getInstance();
+ allPartitions = Collections.EMPTY_SET;
+ leafQueueNames = Collections.EMPTY_SET;
+ preemptableQueues = Collections.EMPTY_MAP;
}
- public ProportionalCapacityPreemptionPolicy(Configuration config,
- RMContext context, CapacityScheduler scheduler) {
- this(config, context, scheduler, SystemClock.getInstance());
- }
-
- public ProportionalCapacityPreemptionPolicy(Configuration config,
- RMContext context, CapacityScheduler scheduler, Clock clock) {
- init(config, context, scheduler);
+ @SuppressWarnings("unchecked")
+ @VisibleForTesting
+ public ProportionalCapacityPreemptionPolicy(RMContext context,
+ CapacityScheduler scheduler, Clock clock) {
+ init(context.getYarnConfiguration(), context, scheduler);
this.clock = clock;
+ allPartitions = Collections.EMPTY_SET;
+ leafQueueNames = Collections.EMPTY_SET;
+ preemptableQueues = Collections.EMPTY_MAP;
}
public void init(Configuration config, RMContext context,
@@ -166,19 +140,45 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
}
rmContext = context;
scheduler = (CapacityScheduler) sched;
- maxIgnoredOverCapacity = config.getDouble(MAX_IGNORED_OVER_CAPACITY, 0.1);
- naturalTerminationFactor =
- config.getDouble(NATURAL_TERMINATION_FACTOR, 0.2);
- maxWaitTime = config.getLong(WAIT_TIME_BEFORE_KILL, 15000);
- monitoringInterval = config.getLong(MONITORING_INTERVAL, 3000);
- percentageClusterPreemptionAllowed =
- config.getFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 0.1);
- observeOnly = config.getBoolean(OBSERVE_ONLY, false);
+ CapacitySchedulerConfiguration csConfig = scheduler.getConfiguration();
+
+ maxIgnoredOverCapacity = csConfig.getDouble(
+ CapacitySchedulerConfiguration.PREEMPTION_MAX_IGNORED_OVER_CAPACITY,
+ CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY);
+
+ naturalTerminationFactor = csConfig.getDouble(
+ CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR,
+ CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR);
+
+ maxWaitTime = csConfig.getLong(
+ CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL,
+ CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL);
+
+ monitoringInterval = csConfig.getLong(
+ CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL,
+ CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MONITORING_INTERVAL);
+
+ percentageClusterPreemptionAllowed = csConfig.getFloat(
+ CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND,
+ CapacitySchedulerConfiguration.DEFAULT_TOTAL_PREEMPTION_PER_ROUND);
+
+ observeOnly = csConfig.getBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY,
+ CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_OBSERVE_ONLY);
+
+ lazyPreempionEnabled = csConfig.getBoolean(
+ CapacitySchedulerConfiguration.LAZY_PREEMPTION_ENALBED,
+ CapacitySchedulerConfiguration.DEFAULT_LAZY_PREEMPTION_ENABLED);
+
rc = scheduler.getResourceCalculator();
nlm = scheduler.getRMContext().getNodeLabelManager();
+
+ // initialize candidates preemption selection policies
+ candidatesSelectionPolicies.add(
+ new FifoCandidatesSelector(this));
}
- @VisibleForTesting
+ @Override
public ResourceCalculator getResourceCalculator() {
return rc;
}
@@ -191,42 +191,37 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
}
@SuppressWarnings("unchecked")
- private void cleanupStaledKillableContainers(Resource cluster,
- Set<String> leafQueueNames) {
- for (String q : leafQueueNames) {
- for (TempQueuePerPartition tq : getQueuePartitions(q)) {
- // When queue's used - killable <= guaranteed and, killable > 0, we need
- // to check if any of killable containers needs to be reverted
- if (Resources.lessThanOrEqual(rc, cluster,
- Resources.subtract(tq.current, tq.killable), tq.idealAssigned)
- && Resources.greaterThan(rc, cluster, tq.killable, Resources.none())) {
- // How many killable resources need to be reverted
- // need-to-revert = already-marked-killable - (current - ideal)
- Resource toBeRevertedFromKillable = Resources.subtract(tq.killable,
- Resources.subtract(tq.current, tq.idealAssigned));
-
- Resource alreadyReverted = Resources.createResource(0);
-
- for (RMContainer c : preemptableEntities.get(q).getKillableContainers(
- tq.partition).values()) {
- if (Resources.greaterThanOrEqual(rc, cluster, alreadyReverted,
- toBeRevertedFromKillable)) {
- break;
- }
-
- if (Resources.greaterThan(rc, cluster,
- Resources.add(alreadyReverted, c.getAllocatedResource()),
- toBeRevertedFromKillable)) {
- continue;
- } else {
- // This container need to be marked to unkillable
- Resources.addTo(alreadyReverted, c.getAllocatedResource());
- rmContext.getDispatcher().getEventHandler().handle(
- new ContainerPreemptEvent(c.getApplicationAttemptId(), c,
- SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE));
- }
+ private void preemptOrkillSelectedContainerAfterWait(
+ Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates) {
+ // preempt (or kill) the selected containers
+ for (Map.Entry<ApplicationAttemptId, Set<RMContainer>> e : selectedCandidates
+ .entrySet()) {
+ ApplicationAttemptId appAttemptId = e.getKey();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Send to scheduler: in app=" + appAttemptId
+ + " #containers-to-be-preemptionCandidates=" + e.getValue().size());
+ }
+ for (RMContainer container : e.getValue()) {
+ // if we tried to preempt this for more than maxWaitTime
+ if (preemptionCandidates.get(container) != null
+ && preemptionCandidates.get(container) + maxWaitTime < clock
+ .getTime()) {
+ // kill it
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(appAttemptId, container,
+ SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
+ preemptionCandidates.remove(container);
+ } else {
+ if (preemptionCandidates.get(container) != null) {
+ // We already updated the information to scheduler earlier, we need
+ // not have to raise another event.
+ continue;
}
-
+ //otherwise just send preemption events
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(appAttemptId, container,
+ SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
+ preemptionCandidates.put(container, clock.getTime());
}
}
}
@@ -234,11 +229,11 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
private void syncKillableContainersFromScheduler() {
// sync preemptable entities from scheduler
- preemptableEntities =
- scheduler.getPreemptionManager().getShallowCopyOfPreemptableEntities();
+ preemptableQueues =
+ scheduler.getPreemptionManager().getShallowCopyOfPreemptableQueues();
killableContainers = new HashSet<>();
- for (Map.Entry<String, PreemptableQueue> entry : preemptableEntities
+ for (Map.Entry<String, PreemptableQueue> entry : preemptableQueues
.entrySet()) {
PreemptableQueue entity = entry.getValue();
for (Map<ContainerId, RMContainer> map : entity.getKillableContainers()
@@ -247,9 +242,34 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
}
}
}
+
+ private void cleanupStaledPreemptionCandidates() {
+ // Keep the preemptionCandidates list clean
+ for (Iterator<RMContainer> i = preemptionCandidates.keySet().iterator();
+ i.hasNext(); ) {
+ RMContainer id = i.next();
+ // garbage collect containers that are irrelevant for preemption
+ if (preemptionCandidates.get(id) + 2 * maxWaitTime < clock.getTime()) {
+ i.remove();
+ }
+ }
+ }
+
+ private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
+ if (q.children == null || q.children.isEmpty()) {
+ return ImmutableSet.of(q.queueName);
+ }
+
+ Set<String> leafQueueNames = new HashSet<>();
+ for (TempQueuePerPartition child : q.children) {
+ leafQueueNames.addAll(getLeafQueueNames(child));
+ }
+
+ return leafQueueNames;
+ }
/**
- * This method selects and tracks containers to be preempted. If a container
+ * This method selects and tracks containers to be preemptionCandidates. If a container
* is in the target list for more than maxWaitTime it is killed.
*
* @param root the root of the CapacityScheduler queue hierarchy
@@ -258,13 +278,17 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
@SuppressWarnings("unchecked")
private void containerBasedPreemptOrKill(CSQueue root,
Resource clusterResources) {
+ // Sync killable containers from scheduler when lazy preemption enabled
+ if (lazyPreempionEnabled) {
+ syncKillableContainersFromScheduler();
+ }
+
// All partitions to look at
- Set<String> allPartitions = new HashSet<>();
- allPartitions.addAll(scheduler.getRMContext()
+ Set<String> partitions = new HashSet<>();
+ partitions.addAll(scheduler.getRMContext()
.getNodeLabelManager().getClusterNodeLabelNames());
- allPartitions.add(RMNodeLabelsManager.NO_LABEL);
-
- syncKillableContainersFromScheduler();
+ partitions.add(RMNodeLabelsManager.NO_LABEL);
+ this.allPartitions = ImmutableSet.copyOf(partitions);
// extract a summary of the queues from scheduler
synchronized (scheduler) {
@@ -277,30 +301,22 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
}
}
+ this.leafQueueNames = ImmutableSet.copyOf(getLeafQueueNames(
+ getQueueByPartition(CapacitySchedulerConfiguration.ROOT,
+ RMNodeLabelsManager.NO_LABEL)));
+
// compute total preemption allowed
Resource totalPreemptionAllowed = Resources.multiply(clusterResources,
percentageClusterPreemptionAllowed);
- Set<String> leafQueueNames = null;
- for (String partition : allPartitions) {
- TempQueuePerPartition tRoot =
- getQueueByPartition(CapacitySchedulerConfiguration.ROOT, partition);
- // compute the ideal distribution of resources among queues
- // updates cloned queues state accordingly
- tRoot.idealAssigned = tRoot.guaranteed;
-
- leafQueueNames =
- recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed);
- }
-
- // remove containers from killable list when we want to preempt less resources
- // from queue.
- cleanupStaledKillableContainers(clusterResources, leafQueueNames);
-
- // based on ideal allocation select containers to be preempted from each
+ // based on ideal allocation select containers to be preemptionCandidates from each
// queue and each application
- Map<ApplicationAttemptId,Set<RMContainer>> toPreempt =
- getContainersToPreempt(leafQueueNames, clusterResources);
+ Map<ApplicationAttemptId, Set<RMContainer>> toPreempt = null;
+ for (PreemptionCandidatesSelector selector :
+ candidatesSelectionPolicies) {
+ toPreempt = selector.selectCandidates(toPreempt,
+ clusterResources, totalPreemptionAllowed);
+ }
if (LOG.isDebugEnabled()) {
logToCSV(new ArrayList<>(leafQueueNames));
@@ -311,582 +327,19 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
return;
}
- // preempt (or kill) the selected containers
- for (Map.Entry<ApplicationAttemptId,Set<RMContainer>> e
- : toPreempt.entrySet()) {
- ApplicationAttemptId appAttemptId = e.getKey();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Send to scheduler: in app=" + appAttemptId
- + " #containers-to-be-preempted=" + e.getValue().size());
- }
- for (RMContainer container : e.getValue()) {
- // if we tried to preempt this for more than maxWaitTime
- if (preempted.get(container) != null &&
- preempted.get(container) + maxWaitTime < clock.getTime()) {
- // mark container killable
- rmContext.getDispatcher().getEventHandler().handle(
- new ContainerPreemptEvent(appAttemptId, container,
- SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
- preempted.remove(container);
- } else {
- if (preempted.get(container) != null) {
- // We already updated the information to scheduler earlier, we need
- // not have to raise another event.
- continue;
- }
- //otherwise just send preemption events
- rmContext.getDispatcher().getEventHandler().handle(
- new ContainerPreemptEvent(appAttemptId, container,
- SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
- preempted.put(container, clock.getTime());
- }
- }
- }
-
- // Keep the preempted list clean
- for (Iterator<RMContainer> i = preempted.keySet().iterator(); i.hasNext();){
- RMContainer id = i.next();
- // garbage collect containers that are irrelevant for preemption
- if (preempted.get(id) + 2 * maxWaitTime < clock.getTime()) {
- i.remove();
- }
- }
- }
-
- /**
- * This method recursively computes the ideal assignment of resources to each
- * level of the hierarchy. This ensures that leafs that are over-capacity but
- * with parents within capacity will not be preempted. Preemptions are allowed
- * within each subtree according to local over/under capacity.
- *
- * @param root the root of the cloned queue hierachy
- * @param totalPreemptionAllowed maximum amount of preemption allowed
- * @return a list of leaf queues updated with preemption targets
- */
- private Set<String> recursivelyComputeIdealAssignment(
- TempQueuePerPartition root, Resource totalPreemptionAllowed) {
- Set<String> leafQueueNames = new HashSet<>();
- if (root.getChildren() != null &&
- root.getChildren().size() > 0) {
- // compute ideal distribution at this level
- computeIdealResourceDistribution(rc, root.getChildren(),
- totalPreemptionAllowed, root.idealAssigned);
- // compute recursively for lower levels and build list of leafs
- for(TempQueuePerPartition t : root.getChildren()) {
- leafQueueNames.addAll(recursivelyComputeIdealAssignment(t,
- totalPreemptionAllowed));
- }
- } else {
- // we are in a leaf nothing to do, just return yourself
- return ImmutableSet.of(root.queueName);
- }
- return leafQueueNames;
- }
+ // TODO: need consider revert killable containers when no more demandings.
+ // Since we could have several selectors to make decisions concurrently.
+ // So computed ideal-allocation varies between different selectors.
+ //
+ // We may need to "score" killable containers and revert the most preferred
+ // containers. The bottom line is, we shouldn't preempt a queue which is already
+ // below its guaranteed resource.
- /**
- * This method computes (for a single level in the tree, passed as a {@code
- * List<TempQueue>}) the ideal assignment of resources. This is done
- * recursively to allocate capacity fairly across all queues with pending
- * demands. It terminates when no resources are left to assign, or when all
- * demand is satisfied.
- *
- * @param rc resource calculator
- * @param queues a list of cloned queues to be assigned capacity to (this is
- * an out param)
- * @param totalPreemptionAllowed total amount of preemption we allow
- * @param tot_guarant the amount of capacity assigned to this pool of queues
- */
- private void computeIdealResourceDistribution(ResourceCalculator rc,
- List<TempQueuePerPartition> queues, Resource totalPreemptionAllowed,
- Resource tot_guarant) {
-
- // qAlloc tracks currently active queues (will decrease progressively as
- // demand is met)
- List<TempQueuePerPartition> qAlloc = new ArrayList<>(queues);
- // unassigned tracks how much resources are still to assign, initialized
- // with the total capacity for this set of queues
- Resource unassigned = Resources.clone(tot_guarant);
-
- // group queues based on whether they have non-zero guaranteed capacity
- Set<TempQueuePerPartition> nonZeroGuarQueues = new HashSet<>();
- Set<TempQueuePerPartition> zeroGuarQueues = new HashSet<>();
-
- for (TempQueuePerPartition q : qAlloc) {
- if (Resources
- .greaterThan(rc, tot_guarant, q.guaranteed, Resources.none())) {
- nonZeroGuarQueues.add(q);
- } else {
- zeroGuarQueues.add(q);
- }
- }
-
- // first compute the allocation as a fixpoint based on guaranteed capacity
- computeFixpointAllocation(rc, tot_guarant, nonZeroGuarQueues, unassigned,
- false);
-
- // if any capacity is left unassigned, distributed among zero-guarantee
- // queues uniformly (i.e., not based on guaranteed capacity, as this is zero)
- if (!zeroGuarQueues.isEmpty()
- && Resources.greaterThan(rc, tot_guarant, unassigned, Resources.none())) {
- computeFixpointAllocation(rc, tot_guarant, zeroGuarQueues, unassigned,
- true);
- }
-
- // based on ideal assignment computed above and current assignment we derive
- // how much preemption is required overall
- Resource totPreemptionNeeded = Resource.newInstance(0, 0);
- for (TempQueuePerPartition t:queues) {
- if (Resources.greaterThan(rc, tot_guarant, t.current, t.idealAssigned)) {
- Resources.addTo(totPreemptionNeeded,
- Resources.subtract(t.current, t.idealAssigned));
- }
- }
-
- // if we need to preempt more than is allowed, compute a factor (0<f<1)
- // that is used to scale down how much we ask back from each queue
- float scalingFactor = 1.0F;
- if (Resources.greaterThan(rc, tot_guarant,
- totPreemptionNeeded, totalPreemptionAllowed)) {
- scalingFactor = Resources.divide(rc, tot_guarant,
- totalPreemptionAllowed, totPreemptionNeeded);
- }
-
- // assign to each queue the amount of actual preemption based on local
- // information of ideal preemption and scaling factor
- for (TempQueuePerPartition t : queues) {
- t.assignPreemption(scalingFactor, rc, tot_guarant);
- }
- if (LOG.isDebugEnabled()) {
- long time = clock.getTime();
- for (TempQueuePerPartition t : queues) {
- LOG.debug(time + ": " + t);
- }
- }
-
- }
-
- /**
- * Given a set of queues compute the fix-point distribution of unassigned
- * resources among them. As pending request of a queue are exhausted, the
- * queue is removed from the set and remaining capacity redistributed among
- * remaining queues. The distribution is weighted based on guaranteed
- * capacity, unless asked to ignoreGuarantee, in which case resources are
- * distributed uniformly.
- */
- private void computeFixpointAllocation(ResourceCalculator rc,
- Resource tot_guarant, Collection<TempQueuePerPartition> qAlloc,
- Resource unassigned, boolean ignoreGuarantee) {
- // Prior to assigning the unused resources, process each queue as follows:
- // If current > guaranteed, idealAssigned = guaranteed + untouchable extra
- // Else idealAssigned = current;
- // Subtract idealAssigned resources from unassigned.
- // If the queue has all of its needs met (that is, if
- // idealAssigned >= current + pending), remove it from consideration.
- // Sort queues from most under-guaranteed to most over-guaranteed.
- TQComparator tqComparator = new TQComparator(rc, tot_guarant);
- PriorityQueue<TempQueuePerPartition> orderedByNeed = new PriorityQueue<>(10,
- tqComparator);
- for (Iterator<TempQueuePerPartition> i = qAlloc.iterator(); i.hasNext();) {
- TempQueuePerPartition q = i.next();
- if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) {
- q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra);
- } else {
- q.idealAssigned = Resources.clone(q.current);
- }
- Resources.subtractFrom(unassigned, q.idealAssigned);
- // If idealAssigned < (current + pending), q needs more resources, so
- // add it to the list of underserved queues, ordered by need.
- Resource curPlusPend = Resources.add(q.current, q.pending);
- if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) {
- orderedByNeed.add(q);
- }
- }
-
- //assign all cluster resources until no more demand, or no resources are left
- while (!orderedByNeed.isEmpty()
- && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) {
- Resource wQassigned = Resource.newInstance(0, 0);
- // we compute normalizedGuarantees capacity based on currently active
- // queues
- resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee);
-
- // For each underserved queue (or set of queues if multiple are equally
- // underserved), offer its share of the unassigned resources based on its
- // normalized guarantee. After the offer, if the queue is not satisfied,
- // place it back in the ordered list of queues, recalculating its place
- // in the order of most under-guaranteed to most over-guaranteed. In this
- // way, the most underserved queue(s) are always given resources first.
- Collection<TempQueuePerPartition> underserved =
- getMostUnderservedQueues(orderedByNeed, tqComparator);
- for (Iterator<TempQueuePerPartition> i = underserved.iterator(); i
- .hasNext();) {
- TempQueuePerPartition sub = i.next();
- Resource wQavail = Resources.multiplyAndNormalizeUp(rc,
- unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1));
- Resource wQidle = sub.offer(wQavail, rc, tot_guarant);
- Resource wQdone = Resources.subtract(wQavail, wQidle);
-
- if (Resources.greaterThan(rc, tot_guarant,
- wQdone, Resources.none())) {
- // The queue is still asking for more. Put it back in the priority
- // queue, recalculating its order based on need.
- orderedByNeed.add(sub);
- }
- Resources.addTo(wQassigned, wQdone);
- }
- Resources.subtractFrom(unassigned, wQassigned);
- }
- }
-
- // Take the most underserved TempQueue (the one on the head). Collect and
- // return the list of all queues that have the same idealAssigned
- // percentage of guaranteed.
- protected Collection<TempQueuePerPartition> getMostUnderservedQueues(
- PriorityQueue<TempQueuePerPartition> orderedByNeed, TQComparator tqComparator) {
- ArrayList<TempQueuePerPartition> underserved = new ArrayList<>();
- while (!orderedByNeed.isEmpty()) {
- TempQueuePerPartition q1 = orderedByNeed.remove();
- underserved.add(q1);
- TempQueuePerPartition q2 = orderedByNeed.peek();
- // q1's pct of guaranteed won't be larger than q2's. If it's less, then
- // return what has already been collected. Otherwise, q1's pct of
- // guaranteed == that of q2, so add q2 to underserved list during the
- // next pass.
- if (q2 == null || tqComparator.compare(q1,q2) < 0) {
- return underserved;
- }
- }
- return underserved;
- }
-
- /**
- * Computes a normalizedGuaranteed capacity based on active queues
- * @param rc resource calculator
- * @param clusterResource the total amount of resources in the cluster
- * @param queues the list of queues to consider
- */
- private void resetCapacity(ResourceCalculator rc, Resource clusterResource,
- Collection<TempQueuePerPartition> queues, boolean ignoreGuar) {
- Resource activeCap = Resource.newInstance(0, 0);
-
- if (ignoreGuar) {
- for (TempQueuePerPartition q : queues) {
- q.normalizedGuarantee = 1.0f / queues.size();
- }
- } else {
- for (TempQueuePerPartition q : queues) {
- Resources.addTo(activeCap, q.guaranteed);
- }
- for (TempQueuePerPartition q : queues) {
- q.normalizedGuarantee = Resources.divide(rc, clusterResource,
- q.guaranteed, activeCap);
- }
- }
- }
-
- private String getPartitionByRMContainer(RMContainer rmContainer) {
- return scheduler.getSchedulerNode(rmContainer.getAllocatedNode())
- .getPartition();
- }
-
- /**
- * Return should we preempt rmContainer. If we should, deduct from
- * <code>resourceToObtainByPartition</code>
- */
- private boolean tryPreemptContainerAndDeductResToObtain(
- Map<String, Resource> resourceToObtainByPartitions,
- RMContainer rmContainer, Resource clusterResource,
- Map<ApplicationAttemptId, Set<RMContainer>> preemptMap) {
- ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId();
-
- // We will not account resource of a container twice or more
- if (preemptMapContains(preemptMap, attemptId, rmContainer)) {
- return false;
- }
-
- String nodePartition = getPartitionByRMContainer(rmContainer);
- Resource toObtainByPartition =
- resourceToObtainByPartitions.get(nodePartition);
-
- if (null != toObtainByPartition
- && Resources.greaterThan(rc, clusterResource, toObtainByPartition,
- Resources.none())) {
- Resources.subtractFrom(toObtainByPartition,
- rmContainer.getAllocatedResource());
- // When we have no more resource need to obtain, remove from map.
- if (Resources.lessThanOrEqual(rc, clusterResource, toObtainByPartition,
- Resources.none())) {
- resourceToObtainByPartitions.remove(nodePartition);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Marked container=" + rmContainer.getContainerId()
- + " in partition=" + nodePartition + " will be preempted");
- }
- // Add to preemptMap
- addToPreemptMap(preemptMap, attemptId, rmContainer);
- return true;
- }
-
- return false;
- }
-
- private boolean preemptMapContains(
- Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
- ApplicationAttemptId attemptId, RMContainer rmContainer) {
- Set<RMContainer> rmContainers;
- if (null == (rmContainers = preemptMap.get(attemptId))) {
- return false;
- }
- return rmContainers.contains(rmContainer);
- }
-
- private void addToPreemptMap(
- Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
- ApplicationAttemptId appAttemptId, RMContainer containerToPreempt) {
- Set<RMContainer> set;
- if (null == (set = preemptMap.get(appAttemptId))) {
- set = new HashSet<>();
- preemptMap.put(appAttemptId, set);
- }
- set.add(containerToPreempt);
- }
-
- /**
- * Based a resource preemption target drop reservations of containers and
- * if necessary select containers for preemption from applications in each
- * over-capacity queue. It uses {@link #NATURAL_TERMINATION_FACTOR} to
- * account for containers that will naturally complete.
- *
- * @param leafQueueNames set of leaf queues to preempt from
- * @param clusterResource total amount of cluster resources
- * @return a map of applciationID to set of containers to preempt
- */
- private Map<ApplicationAttemptId,Set<RMContainer>> getContainersToPreempt(
- Set<String> leafQueueNames, Resource clusterResource) {
-
- Map<ApplicationAttemptId, Set<RMContainer>> preemptMap =
- new HashMap<>();
- List<RMContainer> skippedAMContainerlist = new ArrayList<>();
-
- // Loop all leaf queues
- for (String queueName : leafQueueNames) {
- // check if preemption disabled for the queue
- if (getQueueByPartition(queueName,
- RMNodeLabelsManager.NO_LABEL).preemptionDisabled) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("skipping from queue=" + queueName
- + " because it's a non-preemptable queue");
- }
- continue;
- }
-
- // compute resToObtainByPartition considered inter-queue preemption
- LeafQueue leafQueue = null;
-
- Map<String, Resource> resToObtainByPartition =
- new HashMap<>();
- for (TempQueuePerPartition qT : getQueuePartitions(queueName)) {
- leafQueue = qT.leafQueue;
- // we act only if we are violating balance by more than
- // maxIgnoredOverCapacity
- if (Resources.greaterThan(rc, clusterResource, qT.current,
- Resources.multiply(qT.guaranteed, 1.0 + maxIgnoredOverCapacity))) {
- // we introduce a dampening factor naturalTerminationFactor that
- // accounts for natural termination of containers
- Resource resToObtain =
- Resources.multiply(qT.toBePreempted, naturalTerminationFactor);
- // Only add resToObtain when it >= 0
- if (Resources.greaterThan(rc, clusterResource, resToObtain,
- Resources.none())) {
- resToObtainByPartition.put(qT.partition, resToObtain);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Queue=" + queueName + " partition=" + qT.partition
- + " resource-to-obtain=" + resToObtain);
- }
- }
- qT.actuallyPreempted = Resources.clone(resToObtain);
- } else {
- qT.actuallyPreempted = Resources.none();
- }
- }
-
- synchronized (leafQueue) {
- // go through all ignore-partition-exclusivity containers first to make
- // sure such containers will be preempted first
- Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityContainers =
- leafQueue.getIgnoreExclusivityRMContainers();
- for (String partition : resToObtainByPartition.keySet()) {
- if (ignorePartitionExclusivityContainers.containsKey(partition)) {
- TreeSet<RMContainer> rmContainers =
- ignorePartitionExclusivityContainers.get(partition);
- // We will check container from reverse order, so latter submitted
- // application's containers will be preempted first.
- for (RMContainer c : rmContainers.descendingSet()) {
- boolean preempted =
- tryPreemptContainerAndDeductResToObtain(
- resToObtainByPartition, c, clusterResource, preemptMap);
- if (!preempted) {
- break;
- }
- }
- }
- }
-
- // preempt other containers
- Resource skippedAMSize = Resource.newInstance(0, 0);
- Iterator<FiCaSchedulerApp> desc =
- leafQueue.getOrderingPolicy().getPreemptionIterator();
- while (desc.hasNext()) {
- FiCaSchedulerApp fc = desc.next();
- // When we complete preempt from one partition, we will remove from
- // resToObtainByPartition, so when it becomes empty, we can get no
- // more preemption is needed
- if (resToObtainByPartition.isEmpty()) {
- break;
- }
-
- preemptFrom(fc, clusterResource, resToObtainByPartition,
- skippedAMContainerlist, skippedAMSize, preemptMap);
- }
-
- // Can try preempting AMContainers (still saving atmost
- // maxAMCapacityForThisQueue AMResource's) if more resources are
- // required to be preempted from this Queue.
- Resource maxAMCapacityForThisQueue = Resources.multiply(
- Resources.multiply(clusterResource,
- leafQueue.getAbsoluteCapacity()),
- leafQueue.getMaxAMResourcePerQueuePercent());
-
- preemptAMContainers(clusterResource, preemptMap, skippedAMContainerlist,
- resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue);
- }
- }
-
- return preemptMap;
- }
-
- /**
- * As more resources are needed for preemption, saved AMContainers has to be
- * rescanned. Such AMContainers can be preempted based on resToObtain, but
- * maxAMCapacityForThisQueue resources will be still retained.
- *
- * @param clusterResource
- * @param preemptMap
- * @param skippedAMContainerlist
- * @param skippedAMSize
- * @param maxAMCapacityForThisQueue
- */
- private void preemptAMContainers(Resource clusterResource,
- Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
- List<RMContainer> skippedAMContainerlist,
- Map<String, Resource> resToObtainByPartition, Resource skippedAMSize,
- Resource maxAMCapacityForThisQueue) {
- for (RMContainer c : skippedAMContainerlist) {
- // Got required amount of resources for preemption, can stop now
- if (resToObtainByPartition.isEmpty()) {
- break;
- }
- // Once skippedAMSize reaches down to maxAMCapacityForThisQueue,
- // container selection iteration for preemption will be stopped.
- if (Resources.lessThanOrEqual(rc, clusterResource, skippedAMSize,
- maxAMCapacityForThisQueue)) {
- break;
- }
-
- boolean preempted =
- tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c,
- clusterResource, preemptMap);
- if (preempted) {
- Resources.subtractFrom(skippedAMSize, c.getAllocatedResource());
- }
- }
- skippedAMContainerlist.clear();
- }
-
- /**
- * Given a target preemption for a specific application, select containers
- * to preempt (after unreserving all reservation for that app).
- */
- @SuppressWarnings("unchecked")
- private void preemptFrom(FiCaSchedulerApp app,
- Resource clusterResource, Map<String, Resource> resToObtainByPartition,
- List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
- Map<ApplicationAttemptId, Set<RMContainer>> preemptMap) {
- ApplicationAttemptId appId = app.getApplicationAttemptId();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Looking at application=" + app.getApplicationAttemptId()
- + " resourceToObtain=" + resToObtainByPartition);
- }
-
- // first drop reserved containers towards rsrcPreempt
- List<RMContainer> reservedContainers =
- new ArrayList<>(app.getReservedContainers());
- for (RMContainer c : reservedContainers) {
- if (resToObtainByPartition.isEmpty()) {
- return;
- }
-
- // Try to preempt this container
- tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c,
- clusterResource, preemptMap);
-
- if (!observeOnly) {
- rmContext.getDispatcher().getEventHandler().handle(
- new ContainerPreemptEvent(
- appId, c, SchedulerEventType.KILL_RESERVED_CONTAINER));
- }
- }
-
- // if more resources are to be freed go through all live containers in
- // reverse priority and reverse allocation order and mark them for
- // preemption
- List<RMContainer> liveContainers = new ArrayList<>(app.getLiveContainers());
-
- sortContainers(liveContainers);
-
- for (RMContainer c : liveContainers) {
- if (resToObtainByPartition.isEmpty()) {
- return;
- }
-
- // Skip AM Container from preemption for now.
- if (c.isAMContainer()) {
- skippedAMContainerlist.add(c);
- Resources.addTo(skippedAMSize, c.getAllocatedResource());
- continue;
- }
-
- // Skip already marked to killable containers
- if (killableContainers.contains(c.getContainerId())) {
- continue;
- }
-
- // Try to preempt this container
- tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c,
- clusterResource, preemptMap);
- }
- }
+ // preempt (or kill) the selected containers
+ preemptOrkillSelectedContainerAfterWait(toPreempt);
- /**
- * Compare by reversed priority order first, and then reversed containerId
- * order
- * @param containers
- */
- @VisibleForTesting
- static void sortContainers(List<RMContainer> containers){
- Collections.sort(containers, new Comparator<RMContainer>() {
- @Override
- public int compare(RMContainer a, RMContainer b) {
- Comparator<Priority> c = new org.apache.hadoop.yarn.server
- .resourcemanager.resource.Priority.Comparator();
- int priorityComp = c.compare(b.getContainer().getPriority(),
- a.getContainer().getPriority());
- if (priorityComp != 0) {
- return priorityComp;
- }
- return b.getContainerId().compareTo(a.getContainerId());
- }
- });
+ // cleanup staled preemption candidates
+ cleanupStaledPreemptionCandidates();
}
@Override
@@ -901,7 +354,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
@VisibleForTesting
public Map<RMContainer, Long> getToPreemptContainers() {
- return preempted;
+ return preemptionCandidates;
}
/**
@@ -929,8 +382,8 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
Resource guaranteed = Resources.multiply(partitionResource, absCap);
Resource maxCapacity = Resources.multiply(partitionResource, absMaxCap);
Resource killable = Resources.none();
- if (null != preemptableEntities.get(queueName)) {
- killable = preemptableEntities.get(queueName)
+ if (null != preemptableQueues.get(queueName)) {
+ killable = preemptableQueues.get(queueName)
.getKillableResource(partitionToLookAt);
}
@@ -1023,9 +476,10 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
/**
* Get queue partition by given queueName and partitionName
*/
- private TempQueuePerPartition getQueueByPartition(String queueName,
+ @Override
+ public TempQueuePerPartition getQueueByPartition(String queueName,
String partition) {
- Map<String, TempQueuePerPartition> partitionToQueues = null;
+ Map<String, TempQueuePerPartition> partitionToQueues;
if (null == (partitionToQueues = queueToPartitions.get(queueName))) {
return null;
}
@@ -1035,180 +489,56 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
/**
* Get all queue partitions by given queueName
*/
- private Collection<TempQueuePerPartition> getQueuePartitions(String queueName) {
+ @Override
+ public Collection<TempQueuePerPartition> getQueuePartitions(String queueName) {
if (!queueToPartitions.containsKey(queueName)) {
return null;
}
return queueToPartitions.get(queueName).values();
}
- /**
- * Temporary data-structure tracking resource availability, pending resource
- * need, current utilization. This is per-queue-per-partition data structure
- */
- static class TempQueuePerPartition {
- final String queueName;
- final Resource current;
- final Resource pending;
- final Resource guaranteed;
- final Resource maxCapacity;
- final String partition;
- final Resource killable;
- Resource idealAssigned;
- Resource toBePreempted;
-
- // For logging purpose
- Resource actuallyPreempted;
- Resource untouchableExtra;
- Resource preemptableExtra;
-
- double normalizedGuarantee;
-
- final ArrayList<TempQueuePerPartition> children;
- LeafQueue leafQueue;
- boolean preemptionDisabled;
-
- TempQueuePerPartition(String queueName, Resource current, Resource pending,
- Resource guaranteed, Resource maxCapacity, boolean preemptionDisabled,
- String partition, Resource killableResource) {
- this.queueName = queueName;
- this.current = current;
- this.pending = pending;
- this.guaranteed = guaranteed;
- this.maxCapacity = maxCapacity;
- this.idealAssigned = Resource.newInstance(0, 0);
- this.actuallyPreempted = Resource.newInstance(0, 0);
- this.toBePreempted = Resource.newInstance(0, 0);
- this.normalizedGuarantee = Float.NaN;
- this.children = new ArrayList<>();
- this.untouchableExtra = Resource.newInstance(0, 0);
- this.preemptableExtra = Resource.newInstance(0, 0);
- this.preemptionDisabled = preemptionDisabled;
- this.partition = partition;
- this.killable = killableResource;
- }
-
- public void setLeafQueue(LeafQueue l){
- assert children.size() == 0;
- this.leafQueue = l;
- }
-
- /**
- * When adding a child we also aggregate its pending resource needs.
- * @param q the child queue to add to this queue
- */
- public void addChild(TempQueuePerPartition q) {
- assert leafQueue == null;
- children.add(q);
- Resources.addTo(pending, q.pending);
- }
-
- public ArrayList<TempQueuePerPartition> getChildren(){
- return children;
- }
-
- // This function "accepts" all the resources it can (pending) and return
- // the unused ones
- Resource offer(Resource avail, ResourceCalculator rc,
- Resource clusterResource) {
- Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
- Resources.subtract(maxCapacity, idealAssigned),
- Resource.newInstance(0, 0));
- // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
- Resource accepted =
- Resources.min(rc, clusterResource,
- absMaxCapIdealAssignedDelta,
- Resources.min(rc, clusterResource, avail, Resources.subtract(
- Resources.add(current, pending), idealAssigned)));
- Resource remain = Resources.subtract(avail, accepted);
- Resources.addTo(idealAssigned, accepted);
- return remain;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" NAME: " + queueName)
- .append(" CUR: ").append(current)
- .append(" PEN: ").append(pending)
- .append(" GAR: ").append(guaranteed)
- .append(" NORM: ").append(normalizedGuarantee)
- .append(" IDEAL_ASSIGNED: ").append(idealAssigned)
- .append(" IDEAL_PREEMPT: ").append(toBePreempted)
- .append(" ACTUAL_PREEMPT: ").append(actuallyPreempted)
- .append(" UNTOUCHABLE: ").append(untouchableExtra)
- .append(" PREEMPTABLE: ").append(preemptableExtra)
- .append("\n");
-
- return sb.toString();
- }
+ @Override
+ public CapacityScheduler getScheduler() {
+ return scheduler;
+ }
- public void assignPreemption(float scalingFactor,
- ResourceCalculator rc, Resource clusterResource) {
- if (Resources.greaterThan(rc, clusterResource,
- Resources.subtract(current, killable), idealAssigned)) {
- toBePreempted = Resources.multiply(Resources.subtract(
- Resources.subtract(current, killable), idealAssigned),
- scalingFactor);
- } else {
- toBePreempted = Resource.newInstance(0, 0);
- }
- }
+ @Override
+ public RMContext getRMContext() {
+ return rmContext;
+ }
- void appendLogString(StringBuilder sb) {
- sb.append(queueName).append(", ")
- .append(current.getMemory()).append(", ")
- .append(current.getVirtualCores()).append(", ")
- .append(pending.getMemory()).append(", ")
- .append(pending.getVirtualCores()).append(", ")
- .append(guaranteed.getMemory()).append(", ")
- .append(guaranteed.getVirtualCores()).append(", ")
- .append(idealAssigned.getMemory()).append(", ")
- .append(idealAssigned.getVirtualCores()).append(", ")
- .append(toBePreempted.getMemory()).append(", ")
- .append(toBePreempted.getVirtualCores() ).append(", ")
- .append(actuallyPreempted.getMemory()).append(", ")
- .append(actuallyPreempted.getVirtualCores());
- }
+ @Override
+ public boolean isObserveOnly() {
+ return observeOnly;
+ }
+ @Override
+ public Set<ContainerId> getKillableContainers() {
+ return killableContainers;
}
- static class TQComparator implements Comparator<TempQueuePerPartition> {
- private ResourceCalculator rc;
- private Resource clusterRes;
+ @Override
+ public double getMaxIgnoreOverCapacity() {
+ return maxIgnoredOverCapacity;
+ }
- TQComparator(ResourceCalculator rc, Resource clusterRes) {
- this.rc = rc;
- this.clusterRes = clusterRes;
- }
+ @Override
+ public double getNaturalTerminationFactor() {
+ return naturalTerminationFactor;
+ }
- @Override
- public int compare(TempQueuePerPartition tq1, TempQueuePerPartition tq2) {
- if (getIdealPctOfGuaranteed(tq1) < getIdealPctOfGuaranteed(tq2)) {
- return -1;
- }
- if (getIdealPctOfGuaranteed(tq1) > getIdealPctOfGuaranteed(tq2)) {
- return 1;
- }
- return 0;
- }
+ @Override
+ public Set<String> getLeafQueueNames() {
+ return leafQueueNames;
+ }
- // Calculates idealAssigned / guaranteed
- // TempQueues with 0 guarantees are always considered the most over
- // capacity and therefore considered last for resources.
- private double getIdealPctOfGuaranteed(TempQueuePerPartition q) {
- double pctOver = Integer.MAX_VALUE;
- if (q != null && Resources.greaterThan(
- rc, clusterRes, q.guaranteed, Resources.none())) {
- pctOver =
- Resources.divide(rc, clusterRes, q.idealAssigned, q.guaranteed);
- }
- return (pctOver);
- }
+ @Override
+ public Set<String> getAllPartitions() {
+ return allPartitions;
}
@VisibleForTesting
- public Map<String, Map<String, TempQueuePerPartition>> getQueuePartitions() {
+ Map<String, Map<String, TempQueuePerPartition>> getQueuePartitions() {
return queueToPartitions;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
new file mode 100644
index 0000000..8b01a73
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.util.ArrayList;
+
+/**
+ * Temporary data-structure tracking resource availability, pending resource
+ * need, current utilization. This is per-queue-per-partition data structure
+ */
+public class TempQueuePerPartition {
+ // Following fields are copied from scheduler
+ final String queueName;
+ final Resource current;
+ final Resource pending;
+ final Resource guaranteed;
+ final Resource maxCapacity;
+ final Resource killable;
+ final String partition;
+
+ // Following fields are setted and used by candidate selection policies
+ Resource idealAssigned;
+ Resource toBePreempted;
+ Resource untouchableExtra;
+ Resource preemptableExtra;
+ // For logging purpose
+ Resource actuallyToBePreempted;
+
+ double normalizedGuarantee;
+
+ final ArrayList<TempQueuePerPartition> children;
+ LeafQueue leafQueue;
+ boolean preemptionDisabled;
+
+ TempQueuePerPartition(String queueName, Resource current, Resource pending,
+ Resource guaranteed, Resource maxCapacity, boolean preemptionDisabled,
+ String partition, Resource killable) {
+ this.queueName = queueName;
+ this.current = current;
+ this.pending = pending;
+ this.guaranteed = guaranteed;
+ this.maxCapacity = maxCapacity;
+ this.idealAssigned = Resource.newInstance(0, 0);
+ this.actuallyToBePreempted = Resource.newInstance(0, 0);
+ this.toBePreempted = Resource.newInstance(0, 0);
+ this.normalizedGuarantee = Float.NaN;
+ this.children = new ArrayList<>();
+ this.untouchableExtra = Resource.newInstance(0, 0);
+ this.preemptableExtra = Resource.newInstance(0, 0);
+ this.preemptionDisabled = preemptionDisabled;
+ this.partition = partition;
+ this.killable = killable;
+ }
+
+ public void setLeafQueue(LeafQueue l) {
+ assert children.size() == 0;
+ this.leafQueue = l;
+ }
+
+ /**
+ * When adding a child we also aggregate its pending resource needs.
+ * @param q the child queue to add to this queue
+ */
+ public void addChild(TempQueuePerPartition q) {
+ assert leafQueue == null;
+ children.add(q);
+ Resources.addTo(pending, q.pending);
+ }
+
+ public ArrayList<TempQueuePerPartition> getChildren(){
+ return children;
+ }
+
+ // This function "accepts" all the resources it can (pending) and return
+ // the unused ones
+ Resource offer(Resource avail, ResourceCalculator rc,
+ Resource clusterResource) {
+ Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
+ Resources.subtract(maxCapacity, idealAssigned),
+ Resource.newInstance(0, 0));
+ // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
+ Resource accepted =
+ Resources.min(rc, clusterResource,
+ absMaxCapIdealAssignedDelta,
+ Resources.min(rc, clusterResource, avail, Resources.subtract(
+ Resources.add(current, pending), idealAssigned)));
+ Resource remain = Resources.subtract(avail, accepted);
+ Resources.addTo(idealAssigned, accepted);
+ return remain;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" NAME: " + queueName)
+ .append(" CUR: ").append(current)
+ .append(" PEN: ").append(pending)
+ .append(" GAR: ").append(guaranteed)
+ .append(" NORM: ").append(normalizedGuarantee)
+ .append(" IDEAL_ASSIGNED: ").append(idealAssigned)
+ .append(" IDEAL_PREEMPT: ").append(toBePreempted)
+ .append(" ACTUAL_PREEMPT: ").append(actuallyToBePreempted)
+ .append(" UNTOUCHABLE: ").append(untouchableExtra)
+ .append(" PREEMPTABLE: ").append(preemptableExtra)
+ .append("\n");
+
+ return sb.toString();
+ }
+
+ public void assignPreemption(float scalingFactor, ResourceCalculator rc,
+ Resource clusterResource) {
+ if (Resources.greaterThan(rc, clusterResource,
+ Resources.subtract(current, killable), idealAssigned)) {
+ toBePreempted = Resources.multiply(Resources
+ .subtract(Resources.subtract(current, killable), idealAssigned),
+ scalingFactor);
+ } else {
+ toBePreempted = Resource.newInstance(0, 0);
+ }
+ }
+
+ void appendLogString(StringBuilder sb) {
+ sb.append(queueName).append(", ")
+ .append(current.getMemory()).append(", ")
+ .append(current.getVirtualCores()).append(", ")
+ .append(pending.getMemory()).append(", ")
+ .append(pending.getVirtualCores()).append(", ")
+ .append(guaranteed.getMemory()).append(", ")
+ .append(guaranteed.getVirtualCores()).append(", ")
+ .append(idealAssigned.getMemory()).append(", ")
+ .append(idealAssigned.getVirtualCores()).append(", ")
+ .append(toBePreempted.getMemory()).append(", ")
+ .append(toBePreempted.getVirtualCores() ).append(", ")
+ .append(actuallyToBePreempted.getMemory()).append(", ")
+ .append(actuallyToBePreempted.getVirtualCores());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3729264..88e39de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -1020,4 +1020,49 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public boolean getLazyPreemptionEnabled() {
return getBoolean(LAZY_PREEMPTION_ENALBED, DEFAULT_LAZY_PREEMPTION_ENABLED);
}
+
+ /** If true, run the policy but do not affect the cluster with preemption and
+ * kill events. */
+ public static final String PREEMPTION_OBSERVE_ONLY =
+ "yarn.resourcemanager.monitor.capacity.preemption.observe_only";
+ public static final boolean DEFAULT_PREEMPTION_OBSERVE_ONLY = false;
+
+ /** Time in milliseconds between invocations of this policy */
+ public static final String PREEMPTION_MONITORING_INTERVAL =
+ "yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval";
+ public static final long DEFAULT_PREEMPTION_MONITORING_INTERVAL = 3000L;
+
+ /** Time in milliseconds between requesting a preemption from an application
+ * and killing the container. */
+ public static final String PREEMPTION_WAIT_TIME_BEFORE_KILL =
+ "yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill";
+ public static final long DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL = 15000L;
+
+ /** Maximum percentage of resources preemptionCandidates in a single round. By
+ * controlling this value one can throttle the pace at which containers are
+ * reclaimed from the cluster. After computing the total desired preemption,
+ * the policy scales it back within this limit. */
+ public static final String TOTAL_PREEMPTION_PER_ROUND =
+ "yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round";
+ public static final float DEFAULT_TOTAL_PREEMPTION_PER_ROUND = 0.1f;
+
+ /** Maximum amount of resources above the target capacity ignored for
+ * preemption. This defines a deadzone around the target capacity that helps
+ * prevent thrashing and oscillations around the computed target balance.
+ * High values would slow the time to capacity and (absent natural
+ * completions) it might prevent convergence to guaranteed capacity. */
+ public static final String PREEMPTION_MAX_IGNORED_OVER_CAPACITY =
+ "yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity";
+ public static final float DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY = 0.1f;
+ /**
+ * Given a computed preemption target, account for containers naturally
+ * expiring and preempt only this percentage of the delta. This determines
+ * the rate of geometric convergence into the deadzone ({@link
+ * #PREEMPTION_MAX_IGNORED_OVER_CAPACITY}). For example, a termination factor of 0.5
+ * will reclaim almost 95% of resources within 5 * {@link
+ * #PREEMPTION_WAIT_TIME_BEFORE_KILL}, even absent natural termination. */
+ public static final String PREEMPTION_NATURAL_TERMINATION_FACTOR =
+ "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor";
+ public static final float DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR =
+ 0.2f;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
index 19148d7..fefb56a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
@@ -86,12 +86,6 @@ public class PreemptableQueue {
return res == null ? Resources.none() : res;
}
- @SuppressWarnings("unchecked")
- public Map<ContainerId, RMContainer> getKillableContainers(String partition) {
- Map<ContainerId, RMContainer> map = killableContainers.get(partition);
- return map == null ? Collections.EMPTY_MAP : map;
- }
-
public Map<String, Map<ContainerId, RMContainer>> getKillableContainers() {
return killableContainers;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java
index a9f02a5..76fcd4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptionManager.java
@@ -146,7 +146,7 @@ public class PreemptionManager {
}
}
- public Map<String, PreemptableQueue> getShallowCopyOfPreemptableEntities() {
+ public Map<String, PreemptableQueue> getShallowCopyOfPreemptableQueues() {
try {
readLock.lock();
Map<String, PreemptableQueue> map = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index e9129de..3db4782 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -17,38 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MAX_IGNORED_OVER_CAPACITY;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MONITORING_INTERVAL;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.OBSERVE_ONLY;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.isA;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Random;
-import java.util.StringTokenizer;
-import java.util.TreeSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -63,7 +31,6 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
-import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TempQueuePerPartition;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -95,6 +62,32 @@ import org.mockito.ArgumentMatcher;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.StringTokenizer;
+import java.util.TreeSet;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class TestProportionalCapacityPreemptionPolicy {
static final long TS = 3141592653L;
@@ -105,11 +98,10 @@ public class TestProportionalCapacityPreemptionPolicy {
float setAMResourcePercent = 0.0f;
Random rand = null;
Clock mClock = null;
- Configuration conf = null;
+ CapacitySchedulerConfiguration conf = null;
CapacityScheduler mCS = null;
RMContext rmContext = null;
RMNodeLabelsManager lm = null;
- CapacitySchedulerConfiguration schedConf = null;
EventHandler<SchedulerEvent> mDisp = null;
ResourceCalculator rc = new DefaultResourceCalculator();
Resource clusterResources = null;
@@ -132,7 +124,7 @@ public class TestProportionalCapacityPreemptionPolicy {
AMCONTAINER(0), CONTAINER(1), LABELEDCONTAINER(2);
int value;
- private priority(int value) {
+ priority(int value) {
this.value = value;
}
@@ -146,12 +138,17 @@ public class TestProportionalCapacityPreemptionPolicy {
@Before
@SuppressWarnings("unchecked")
public void setup() {
- conf = new Configuration(false);
- conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
- conf.setLong(MONITORING_INTERVAL, 3000);
+ conf = new CapacitySchedulerConfiguration(new Configuration(false));
+ conf.setLong(
+ CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
+ conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL,
+ 3000);
// report "ideal" preempt
- conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
- conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
+ conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND,
+ 1.0f);
+ conf.setFloat(
+ CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR,
+ 1.0f);
conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
@@ -164,8 +161,7 @@ public class TestProportionalCapacityPreemptionPolicy {
mCS = mock(CapacityScheduler.class);
when(mCS.getResourceCalculator()).thenReturn(rc);
lm = mock(RMNodeLabelsManager.class);
- schedConf = new CapacitySchedulerConfiguration();
- when(mCS.getConfiguration()).thenReturn(schedConf);
+ when(mCS.getConfiguration()).thenReturn(conf);
rmContext = mock(RMContext.class);
when(mCS.getRMContext()).thenReturn(rmContext);
when(mCS.getPreemptionManager()).thenReturn(new PreemptionManager());
@@ -271,7 +267,9 @@ public class TestProportionalCapacityPreemptionPolicy {
{ -1, 1, 1, 1 }, // req granularity
{ 3, 0, 0, 0 }, // subqueues
};
- conf.setLong(WAIT_TIME_BEFORE_KILL, killTime);
+ conf.setLong(
+ CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL,
+ killTime);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
// ensure all pending rsrc from A get preempted from other queues
@@ -308,7 +306,9 @@ public class TestProportionalCapacityPreemptionPolicy {
{ -1, 1, 1, 1 }, // req granularity
{ 3, 0, 0, 0 }, // subqueues
};
- conf.setFloat(MAX_IGNORED_OVER_CAPACITY, (float) 0.1);
+ conf.setFloat(
+ CapacitySchedulerConfiguration.PREEMPTION_MAX_IGNORED_OVER_CAPACITY,
+ (float) 0.1);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// ignore 10% overcapacity to avoid jitter
@@ -330,7 +330,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ 3, 0, 0, 0 }, // subqueues
};
- schedConf.setPreemptionDisabled("root.queueB", true);
+ conf.setPreemptionDisabled("root.queueB", true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
@@ -343,7 +343,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// event handler will count only events from the following test and not the
// previous one.
setup();
- schedConf.setPreemptionDisabled("root.queueB", false);
+ conf.setPreemptionDisabled("root.queueB", false);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
@@ -382,7 +382,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// Need to call setup() again to reset mDisp
setup();
// Turn off preemption for queueB and it's children
- schedConf.setPreemptionDisabled("root.queueA.queueB", true);
+ conf.setPreemptionDisabled("root.queueA.queueB", true);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
ApplicationAttemptId expectedAttemptOnQueueC =
@@ -429,7 +429,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// Need to call setup() again to reset mDisp
setup();
// Turn off preemption for queueB(appA)
- schedConf.setPreemptionDisabled("root.queueA.queueB", true);
+ conf.setPreemptionDisabled("root.queueA.queueB", true);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
// Now that queueB(appA) is not preemptable, verify that resources come
@@ -439,8 +439,8 @@ public class TestProportionalCapacityPreemptionPolicy {
setup();
// Turn off preemption for two of the 3 queues with over-capacity.
- schedConf.setPreemptionDisabled("root.queueD.queueE", true);
- schedConf.setPreemptionDisabled("root.queueA.queueB", true);
+ conf.setPreemptionDisabled("root.queueD.queueE", true);
+ conf.setPreemptionDisabled("root.queueA.queueB", true);
ProportionalCapacityPreemptionPolicy policy3 = buildPolicy(qData);
policy3.editSchedule();
@@ -481,7 +481,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// Turn off preemption for queueA and it's children. queueF(appC)'s request
// should starve.
setup(); // Call setup() to reset mDisp
- schedConf.setPreemptionDisabled("root.queueA", true);
+ conf.setPreemptionDisabled("root.queueA", true);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA))); // queueC
@@ -505,7 +505,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ -1, -1, 1, 1, 1, -1, 1, 1, 1 }, // req granularity
{ 2, 3, 0, 0, 0, 3, 0, 0, 0 }, // subqueues
};
- schedConf.setPreemptionDisabled("root.queueA.queueC", true);
+ conf.setPreemptionDisabled("root.queueA.queueC", true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// Although queueC(appB) is way over capacity and is untouchable,
@@ -529,7 +529,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ 3, 2, 0, 0, 2, 0, 0, 2, 0, 0 }, // subqueues
};
- schedConf.setPreemptionDisabled("root", true);
+ conf.setPreemptionDisabled("root", true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// All queues should be non-preemptable, so request should starve.
@@ -556,7 +556,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ 2, 2, 0, 0, 2, 0, 0 }, // subqueues
};
// QueueE inherits non-preemption from QueueD
- schedConf.setPreemptionDisabled("root.queueD", true);
+ conf.setPreemptionDisabled("root.queueD", true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// appC is running on QueueE. QueueE is over absMaxCap, but is not
@@ -596,7 +596,10 @@ public class TestProportionalCapacityPreemptionPolicy {
{ -1, 1, 1, 0 }, // req granularity
{ 3, 0, 0, 0 }, // subqueues
};
- conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 0.1);
+ conf.setFloat(
+ CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR,
+ (float) 0.1);
+
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// ignore 10% imbalance between over-capacity queues
@@ -616,7 +619,10 @@ public class TestProportionalCapacityPreemptionPolicy {
{ -1, 1, 1, 0 }, // req granularity
{ 3, 0, 0, 0 }, // subqueues
};
- conf.setBoolean(OBSERVE_ONLY, true);
+ conf.setBoolean(CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY,
+ true);
+ when(mCS.getConfiguration()).thenReturn(
+ new CapacitySchedulerConfiguration(conf));
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// verify even severe imbalance not affected
@@ -735,7 +741,7 @@ public class TestProportionalCapacityPreemptionPolicy {
containers.add(rm4);
// sort them
- ProportionalCapacityPreemptionPolicy.sortContainers(containers);
+ FifoCandidatesSelector.sortContainers(containers);
// verify the "priority"-first, "reverse container-id"-second
// ordering is enforced correctly
@@ -957,7 +963,7 @@ public class TestProportionalCapacityPreemptionPolicy {
ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) {
ProportionalCapacityPreemptionPolicy policy = new ProportionalCapacityPreemptionPolicy(
- conf, rmContext, mCS, mClock);
+ rmContext, mCS, mClock);
clusterResources = Resource.newInstance(
leafAbsCapacities(qData[0], qData[7]), 0);
ParentQueue mRoot = buildMockRootQueue(rand, qData);
@@ -968,18 +974,13 @@ public class TestProportionalCapacityPreemptionPolicy {
}
ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData,
- String[][] resData) {
- return buildPolicy(qData, resData, false);
- }
-
- ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData,
String[][] resData, boolean useDominantResourceCalculator) {
if (useDominantResourceCalculator) {
when(mCS.getResourceCalculator()).thenReturn(
new DominantResourceCalculator());
}
ProportionalCapacityPreemptionPolicy policy =
- new ProportionalCapacityPreemptionPolicy(conf, rmContext, mCS, mClock);
+ new ProportionalCapacityPreemptionPolicy(rmContext, mCS, mClock);
clusterResources = leafAbsCapacities(parseResourceDetails(resData[0]),
qData[2]);
ParentQueue mRoot = buildMockRootQueue(rand, resData, qData);
@@ -1124,7 +1125,7 @@ public class TestProportionalCapacityPreemptionPolicy {
String qName = "";
while(tokenizer.hasMoreTokens()) {
qName += tokenizer.nextToken();
- preemptionDisabled = schedConf.getPreemptionDisabled(qName, preemptionDisabled);
+ preemptionDisabled = conf.getPreemptionDisabled(qName, preemptionDisabled);
qName += ".";
}
return preemptionDisabled;
[14/50] [abbrv] hadoop git commit: HADOOP-12916. Allow RPC
scheduler/callqueue backoff using response times. Contributed by Xiaoyu Yao.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index 99bfc61..2ebd1c5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -43,8 +43,10 @@ import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.MetricsAsserts;
import org.apache.hadoop.test.MockitoUtil;
+import org.apache.log4j.Level;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -956,7 +958,7 @@ public class TestRPC extends TestRpcBase {
}
/**
- * Test RPC backoff.
+ * Test RPC backoff by queue full.
*/
@Test (timeout=30000)
public void testClientBackOff() throws Exception {
@@ -969,7 +971,7 @@ public class TestRPC extends TestRpcBase {
final ExecutorService executorService =
Executors.newFixedThreadPool(numClients);
conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- conf.setBoolean(CommonConfigurationKeys.IPC_CALLQUEUE_NAMESPACE +
+ conf.setBoolean(CommonConfigurationKeys.IPC_NAMESPACE +
".0." + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true);
RPC.Builder builder = newServerBuilder(conf)
.setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
@@ -1019,6 +1021,92 @@ public class TestRPC extends TestRpcBase {
}
/**
+ * Test RPC backoff by response time of each priority level.
+ */
+ @Test (timeout=30000)
+ public void testClientBackOffByResponseTime() throws Exception {
+ Server server;
+ final TestRpcService proxy;
+ boolean succeeded = false;
+ final int numClients = 1;
+ final int queueSizePerHandler = 3;
+
+ GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG);
+ GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG);
+
+ final List<Future<Void>> res = new ArrayList<Future<Void>>();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(numClients);
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0.";
+ conf.setBoolean(ns + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true);
+ conf.setStrings(ns + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY,
+ "org.apache.hadoop.ipc.FairCallQueue");
+ conf.setStrings(ns + CommonConfigurationKeys.IPC_SCHEDULER_IMPL_KEY,
+ "org.apache.hadoop.ipc.DecayRpcScheduler");
+ conf.setInt(ns + CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_KEY,
+ 2);
+ conf.setBoolean(ns +
+ DecayRpcScheduler.IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY,
+ true);
+ // set a small thresholds 2s and 4s for level 0 and level 1 for testing
+ conf.set(ns +
+ DecayRpcScheduler.IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY
+ , "2s, 4s");
+
+ // Set max queue size to 3 so that 2 calls from the test won't trigger
+ // back off because the queue is full.
+ RPC.Builder builder = newServerBuilder(conf)
+ .setQueueSizePerHandler(queueSizePerHandler).setNumHandlers(1)
+ .setVerbose(true);
+ server = setupTestServer(builder);
+
+ @SuppressWarnings("unchecked")
+ CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox
+ .getInternalState(server, "callQueue"));
+ Whitebox.setInternalState(server, "callQueue", spy);
+
+ Exception lastException = null;
+ proxy = getClient(addr, conf);
+ try {
+ // start a sleep RPC call that sleeps 3s.
+ for (int i = 0; i < numClients; i++) {
+ res.add(executorService.submit(
+ new Callable<Void>() {
+ @Override
+ public Void call() throws ServiceException, InterruptedException {
+ proxy.sleep(null, newSleepRequest(3000));
+ return null;
+ }
+ }));
+ verify(spy, timeout(500).times(i + 1)).offer(Mockito.<Call>anyObject());
+ }
+ // Start another sleep RPC call and verify the call is backed off due to
+ // avg response time(3s) exceeds threshold (2s).
+ try {
+ // wait for the 1st response time update
+ Thread.sleep(5500);
+ proxy.sleep(null, newSleepRequest(100));
+ } catch (ServiceException e) {
+ RemoteException re = (RemoteException) e.getCause();
+ IOException unwrapExeption = re.unwrapRemoteException();
+ if (unwrapExeption instanceof RetriableException) {
+ succeeded = true;
+ } else {
+ lastException = unwrapExeption;
+ }
+ }
+ } finally {
+ executorService.shutdown();
+ stop(server, proxy);
+ }
+ if (lastException != null) {
+ LOG.error("Last received non-RetriableException:", lastException);
+ }
+ assertTrue("RetriableException not received", succeeded);
+ }
+
+ /**
* Test RPC timeout.
*/
@Test(timeout=30000)
[47/50] [abbrv] hadoop git commit: HADOOP-12672. RPC timeout should
not override IPC ping interval (iwasakims)
Posted by ae...@apache.org.
HADOOP-12672. RPC timeout should not override IPC ping interval (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85ec5573
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85ec5573
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85ec5573
Branch: refs/heads/HDFS-1312
Commit: 85ec5573eb9fd746a9295ecc6fe1ae683073aaf5
Parents: 0005816
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Apr 6 03:22:48 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Apr 6 03:22:48 2016 +0900
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Client.java | 57 +++++++++++++----
.../src/main/resources/core-default.xml | 9 ++-
.../java/org/apache/hadoop/ipc/TestRPC.java | 67 ++++++++++++++++++--
.../hadoop/hdfs/client/impl/DfsClientConf.java | 2 +-
4 files changed, 108 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec5573/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 7e6c7e3..fb11cb7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -239,14 +239,33 @@ public class Client {
*
* @param conf Configuration
* @return the timeout period in milliseconds. -1 if no timeout value is set
+ * @deprecated use {@link #getRpcTimeout(Configuration)} instead
*/
+ @Deprecated
final public static int getTimeout(Configuration conf) {
+ int timeout = getRpcTimeout(conf);
+ if (timeout > 0) {
+ return timeout;
+ }
if (!conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY,
CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT)) {
return getPingInterval(conf);
}
return -1;
}
+
+ /**
+ * The time after which a RPC will timeout.
+ *
+ * @param conf Configuration
+ * @return the timeout period in milliseconds.
+ */
+ public static final int getRpcTimeout(Configuration conf) {
+ int timeout =
+ conf.getInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_DEFAULT);
+ return (timeout < 0) ? 0 : timeout;
+ }
/**
* set the connection timeout value in configuration
*
@@ -386,7 +405,7 @@ public class Client {
private Socket socket = null; // connected socket
private DataInputStream in;
private DataOutputStream out;
- private int rpcTimeout;
+ private final int rpcTimeout;
private int maxIdleTime; //connections will be culled if it was idle for
//maxIdleTime msecs
private final RetryPolicy connectionRetryPolicy;
@@ -394,8 +413,9 @@ public class Client {
private int maxRetriesOnSocketTimeouts;
private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
private final boolean tcpLowLatency; // if T then use low-delay QoS
- private boolean doPing; //do we need to send ping message
- private int pingInterval; // how often sends ping to the server in msecs
+ private final boolean doPing; //do we need to send ping message
+ private final int pingInterval; // how often sends ping to the server
+ private final int soTimeout; // used by ipc ping and rpc timeout
private ByteArrayOutputStream pingRequest; // ping message
// currently active calls
@@ -434,6 +454,14 @@ public class Client {
pingHeader.writeDelimitedTo(pingRequest);
}
this.pingInterval = remoteId.getPingInterval();
+ if (rpcTimeout > 0) {
+ // effective rpc timeout is rounded up to multiple of pingInterval
+ // if pingInterval < rpcTimeout.
+ this.soTimeout = (doPing && pingInterval < rpcTimeout) ?
+ pingInterval : rpcTimeout;
+ } else {
+ this.soTimeout = pingInterval;
+ }
this.serviceClass = serviceClass;
if (LOG.isDebugEnabled()) {
LOG.debug("The ping interval is " + this.pingInterval + " ms.");
@@ -484,12 +512,12 @@ public class Client {
/* Process timeout exception
* if the connection is not going to be closed or
- * is not configured to have a RPC timeout, send a ping.
- * (if rpcTimeout is not set to be 0, then RPC should timeout.
- * otherwise, throw the timeout exception.
+ * the RPC is not timed out yet, send a ping.
*/
- private void handleTimeout(SocketTimeoutException e) throws IOException {
- if (shouldCloseConnection.get() || !running.get() || rpcTimeout > 0) {
+ private void handleTimeout(SocketTimeoutException e, int waiting)
+ throws IOException {
+ if (shouldCloseConnection.get() || !running.get() ||
+ (0 < rpcTimeout && rpcTimeout <= waiting)) {
throw e;
} else {
sendPing();
@@ -503,11 +531,13 @@ public class Client {
*/
@Override
public int read() throws IOException {
+ int waiting = 0;
do {
try {
return super.read();
} catch (SocketTimeoutException e) {
- handleTimeout(e);
+ waiting += soTimeout;
+ handleTimeout(e, waiting);
}
} while (true);
}
@@ -520,11 +550,13 @@ public class Client {
*/
@Override
public int read(byte[] buf, int off, int len) throws IOException {
+ int waiting = 0;
do {
try {
return super.read(buf, off, len);
} catch (SocketTimeoutException e) {
- handleTimeout(e);
+ waiting += soTimeout;
+ handleTimeout(e, waiting);
}
} while (true);
}
@@ -632,10 +664,7 @@ public class Client {
}
NetUtils.connect(this.socket, server, connectionTimeout);
- if (rpcTimeout > 0) {
- pingInterval = rpcTimeout; // rpcTimeout overwrites pingInterval
- }
- this.socket.setSoTimeout(pingInterval);
+ this.socket.setSoTimeout(soTimeout);
return;
} catch (ConnectTimeoutException toe) {
/* Check for an address change and update the local reference.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec5573/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index d6e2bcd..b3436da 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1054,7 +1054,7 @@
<value>true</value>
<description>Send a ping to the server when timeout on reading the response,
if set to true. If no failure is detected, the client retries until at least
- a byte is read.
+ a byte is read or the time given by ipc.client.rpc-timeout.ms is passed.
</description>
</property>
@@ -1071,10 +1071,9 @@
<name>ipc.client.rpc-timeout.ms</name>
<value>0</value>
<description>Timeout on waiting response from server, in milliseconds.
- Currently this timeout works only when ipc.client.ping is set to true
- because it uses the same facilities with IPC ping.
- The timeout overrides the ipc.ping.interval and client will throw exception
- instead of sending ping when the interval is passed.
+ If ipc.client.ping is set to true and this rpc-timeout is greater than
+ the value of ipc.ping.interval, the effective value of the rpc-timeout is
+ rounded up to multiple of ipc.ping.interval.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec5573/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index 2ebd1c5..4eb3bea 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -1118,14 +1118,67 @@ public class TestRPC extends TestRpcBase {
.setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
server = setupTestServer(builder);
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
try {
- proxy = getClient(addr, conf);
- proxy.sleep(null, newSleepRequest(3000));
- fail("RPC should time out.");
- } catch (ServiceException e) {
- assertTrue(e.getCause() instanceof SocketTimeoutException);
- LOG.info("got expected timeout.", e);
+ // Test RPC timeout with default ipc.client.ping.
+ try {
+ Configuration c = new Configuration(conf);
+ c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
+ proxy = getClient(addr, c);
+ proxy.sleep(null, newSleepRequest(3000));
+ fail("RPC should time out.");
+ } catch (ServiceException e) {
+ assertTrue(e.getCause() instanceof SocketTimeoutException);
+ LOG.info("got expected timeout.", e);
+ }
+
+ // Test RPC timeout when ipc.client.ping is false.
+ try {
+ Configuration c = new Configuration(conf);
+ c.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, false);
+ c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
+ proxy = getClient(addr, c);
+ proxy.sleep(null, newSleepRequest(3000));
+ fail("RPC should time out.");
+ } catch (ServiceException e) {
+ assertTrue(e.getCause() instanceof SocketTimeoutException);
+ LOG.info("got expected timeout.", e);
+ }
+
+ // Test negative timeout value.
+ try {
+ Configuration c = new Configuration(conf);
+ c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, -1);
+ proxy = getClient(addr, c);
+ proxy.sleep(null, newSleepRequest(2000));
+ } catch (ServiceException e) {
+ LOG.info("got unexpected exception.", e);
+ fail("RPC should not time out.");
+ }
+
+ // Test RPC timeout greater than ipc.ping.interval.
+ try {
+ Configuration c = new Configuration(conf);
+ c.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
+ c.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, 800);
+ c.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
+ proxy = getClient(addr, c);
+
+ try {
+ // should not time out because effective rpc-timeout is
+ // multiple of ping interval: 1600 (= 800 * (1000 / 800 + 1))
+ proxy.sleep(null, newSleepRequest(1300));
+ } catch (ServiceException e) {
+ LOG.info("got unexpected exception.", e);
+ fail("RPC should not time out.");
+ }
+
+ proxy.sleep(null, newSleepRequest(2000));
+ fail("RPC should time out.");
+ } catch (ServiceException e) {
+ assertTrue(e.getCause() instanceof SocketTimeoutException);
+ LOG.info("got expected timeout.", e);
+ }
+
} finally {
stop(server, proxy);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec5573/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index eae3c8e..883a5bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -141,7 +141,7 @@ public class DfsClientConf {
public DfsClientConf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
- hdfsTimeout = Client.getTimeout(conf);
+ hdfsTimeout = Client.getRpcTimeout(conf);
maxRetryAttempts = conf.getInt(
Retry.MAX_ATTEMPTS_KEY,
[23/50] [abbrv] hadoop git commit: Revert "YARN-4857. Add missing
default configuration regarding preemption of CapacityScheduler. Contributed
by Kai Sasaki."
Posted by ae...@apache.org.
Revert "YARN-4857. Add missing default configuration regarding preemption of CapacityScheduler. Contributed by Kai Sasaki."
This reverts commit 0064cba169d1bb761f6e81ee86830be598d7c500.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3488c4f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3488c4f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3488c4f2
Branch: refs/heads/HDFS-1312
Commit: 3488c4f2c9767684eb1007bb00250f474c06d5d8
Parents: a8d8b80
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Apr 1 12:20:40 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Apr 1 12:20:40 2016 +0530
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 58 --------------------
1 file changed, 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3488c4f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index cb3c73a..506cf3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -908,64 +908,6 @@
<value>600000</value>
</property>
- <property>
- <description>
- If true, run the policy but do not affect the cluster with preemption and kill events.
- </description>
- <name>yarn.resourcemanager.monitor.capacity.preemption.observe_only</name>
- <value>false</value>
- </property>
-
- <property>
- <description>
- Time in milliseconds between invocations of this ProportionalCapacityPreemptionPolicy
- policy.
- </description>
- <name>yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval</name>
- <value>3000</value>
- </property>
-
- <property>
- <description>
- Time in milliseconds between requesting a preemption from an application and killing
- the container.
- </description>
- <name>yarn.resourcemanager.monitor.capacity.preemption.max_wait_before_kill</name>
- <value>15000</value>
- </property>
-
- <property>
- <description>
- Maximum percentage of resources preempted in a single round. By controlling this valueone
- can throttle the pace at which containers are reclaimed from the cluster. After computing
- the total desired preemption, the policy scales it back within this limit.
- </description>
- <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
- <value>0.1</value>
- </property>
-
- <property>
- <description>
- Maximum amount of resources above the target capacity ignored for preemption.
- This defines a deadzone around the target capacity that helps prevent thrashing and
- oscillations around the computed target balance. High values would slow the time to capacity
- and (absent natural.completions) it might prevent convergence to guaranteed capacity.
- </description>
- <name>yarn.resourcemanager.monitor.capacity.preemption.max_ignored_over_capacity</name>
- <value>0.1</value>
- </property>
-
- <property>
- <description>
- Given a computed preemption target, account for containers naturally expiring and preempt
- only this percentage of the delta. This determines the rate of geometric convergence into
- the deadzone (MAX_IGNORED_OVER_CAPACITY). For example, a termination factor of 0.5 will reclaim
- almost 95% of resources within 5 * #WAIT_TIME_BEFORE_KILL, even absent natural termination.
- </description>
- <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
- <value>0.2</value>
- </property>
-
<!-- Node Manager Configuration -->
<property>
[17/50] [abbrv] hadoop git commit: YARN-4639. Remove dead code in
TestDelegationTokenRenewer added in YARN-3055 (templedf via rkanter)
Posted by ae...@apache.org.
YARN-4639. Remove dead code in TestDelegationTokenRenewer added in YARN-3055 (templedf via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a021471
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a021471
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a021471
Branch: refs/heads/HDFS-1312
Commit: 7a021471c376ce846090fbd1a315266bada048d4
Parents: 6d67420
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Mar 31 13:09:09 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Mar 31 13:09:09 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/security/TestDelegationTokenRenewer.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a021471/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index d85e928..1bfac8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -1090,9 +1090,6 @@ public class TestDelegationTokenRenewer {
new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
nm1.registerNode();
- //MyFS fs = (MyFS)FileSystem.get(conf);
- //MyToken token1 = fs.getDelegationToken("user123");
-
// create Token1:
Text userText1 = new Text("user");
DelegationTokenIdentifier dtId1 =
[07/50] [abbrv] hadoop git commit: HADOOP-12886. Exclude weak ciphers
in SSLFactory through ssl-server.xml. Contributed by Wei-Chiu Chuang.
Posted by ae...@apache.org.
HADOOP-12886. Exclude weak ciphers in SSLFactory through ssl-server.xml. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4fc609d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4fc609d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4fc609d
Branch: refs/heads/HDFS-1312
Commit: e4fc609d5d3739b7809057954c5233cfd1d1117b
Parents: 37e23ce
Author: Zhe ZHang <ze...@linkedin.com>
Authored: Wed Mar 30 14:13:11 2016 -0700
Committer: Zhe ZHang <ze...@linkedin.com>
Committed: Wed Mar 30 14:13:11 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/security/ssl/SSLFactory.java | 42 +++++-
.../hadoop/security/ssl/TestSSLFactory.java | 139 ++++++++++++++++++-
2 files changed, 175 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4fc609d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index ea65848..95cba80 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
import javax.net.ssl.HostnameVerifier;
@@ -34,6 +36,11 @@ import javax.net.ssl.SSLSocketFactory;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
/**
* Factory that creates SSLEngine and SSLSocketFactory instances using
@@ -48,6 +55,7 @@ import java.security.GeneralSecurityException;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class SSLFactory implements ConnectionConfigurator {
+ static final Logger LOG = LoggerFactory.getLogger(SSLFactory.class);
@InterfaceAudience.Private
public static enum Mode { CLIENT, SERVER }
@@ -60,7 +68,7 @@ public class SSLFactory implements ConnectionConfigurator {
"hadoop.ssl.client.conf";
public static final String SSL_SERVER_CONF_KEY =
"hadoop.ssl.server.conf";
- public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509";
+ public static final String SSLCERTIFICATE = IBM_JAVA?"ibmX509":"SunX509";
public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false;
@@ -71,6 +79,8 @@ public class SSLFactory implements ConnectionConfigurator {
"hadoop.ssl.enabled.protocols";
public static final String DEFAULT_SSL_ENABLED_PROTOCOLS =
"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2";
+ public static final String SSL_SERVER_EXCLUDE_CIPHER_LIST =
+ "ssl.server.exclude.cipher.list";
private Configuration conf;
private Mode mode;
@@ -80,6 +90,7 @@ public class SSLFactory implements ConnectionConfigurator {
private KeyStoresFactory keystoresFactory;
private String[] enabledProtocols = null;
+ private List<String> excludeCiphers;
/**
* Creates an SSLFactory.
@@ -105,6 +116,14 @@ public class SSLFactory implements ConnectionConfigurator {
enabledProtocols = conf.getStrings(SSL_ENABLED_PROTOCOLS,
DEFAULT_SSL_ENABLED_PROTOCOLS);
+ String excludeCiphersConf =
+ sslConf.get(SSL_SERVER_EXCLUDE_CIPHER_LIST, "");
+ if (excludeCiphersConf.isEmpty()) {
+ excludeCiphers = new LinkedList<String>();
+ } else {
+ LOG.debug("will exclude cipher suites: {}", excludeCiphersConf);
+ excludeCiphers = Arrays.asList(excludeCiphersConf.split(","));
+ }
}
private Configuration readSSLConfiguration(Mode mode) {
@@ -195,11 +214,32 @@ public class SSLFactory implements ConnectionConfigurator {
} else {
sslEngine.setUseClientMode(false);
sslEngine.setNeedClientAuth(requireClientCert);
+ disableExcludedCiphers(sslEngine);
}
sslEngine.setEnabledProtocols(enabledProtocols);
return sslEngine;
}
+ private void disableExcludedCiphers(SSLEngine sslEngine) {
+ String[] cipherSuites = sslEngine.getEnabledCipherSuites();
+
+ ArrayList<String> defaultEnabledCipherSuites =
+ new ArrayList<String>(Arrays.asList(cipherSuites));
+ Iterator iterator = excludeCiphers.iterator();
+
+ while(iterator.hasNext()) {
+ String cipherName = (String)iterator.next();
+ if(defaultEnabledCipherSuites.contains(cipherName)) {
+ defaultEnabledCipherSuites.remove(cipherName);
+ LOG.debug("Disabling cipher suite {}.", cipherName);
+ }
+ }
+
+ cipherSuites = defaultEnabledCipherSuites.toArray(
+ new String[defaultEnabledCipherSuites.size()]);
+ sslEngine.setEnabledCipherSuites(cipherSuites);
+ }
+
/**
* Returns a configured SSLServerSocketFactory.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4fc609d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
index 004888c..b8a09ed 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java
@@ -17,24 +17,31 @@
*/
package org.apache.hadoop.security.ssl;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLSession;
import java.io.File;
import java.net.URL;
+import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
import java.security.KeyPair;
import java.security.cert.X509Certificate;
@@ -42,13 +49,21 @@ import java.util.Collections;
import java.util.Map;
public class TestSSLFactory {
-
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestSSLFactory.class);
private static final String BASEDIR =
System.getProperty("test.build.dir", "target/test-dir") + "/" +
TestSSLFactory.class.getSimpleName();
private static final String KEYSTORES_DIR =
new File(BASEDIR).getAbsolutePath();
private String sslConfsDir;
+ private static final String excludeCiphers = "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ + "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ + "SSL_RSA_WITH_DES_CBC_SHA,"
+ + "SSL_DHE_RSA_WITH_DES_CBC_SHA,"
+ + "SSL_RSA_EXPORT_WITH_RC4_40_MD5,"
+ + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ + "SSL_RSA_WITH_RC4_128_MD5";
@BeforeClass
public static void setUp() throws Exception {
@@ -62,7 +77,7 @@ public class TestSSLFactory {
throws Exception {
Configuration conf = new Configuration();
KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf,
- clientCert, trustStore);
+ clientCert, trustStore, excludeCiphers);
return conf;
}
@@ -125,6 +140,120 @@ public class TestSSLFactory {
serverMode(true, false);
}
+ private void runDelegatedTasks(SSLEngineResult result, SSLEngine engine)
+ throws Exception {
+ Runnable runnable;
+ if (result.getHandshakeStatus() ==
+ SSLEngineResult.HandshakeStatus.NEED_TASK) {
+ while ((runnable = engine.getDelegatedTask()) != null) {
+ LOG.info("running delegated task...");
+ runnable.run();
+ }
+ SSLEngineResult.HandshakeStatus hsStatus = engine.getHandshakeStatus();
+ if (hsStatus == SSLEngineResult.HandshakeStatus.NEED_TASK) {
+ throw new Exception("handshake shouldn't need additional tasks");
+ }
+ }
+ }
+
+ private static boolean isEngineClosed(SSLEngine engine) {
+ return engine.isOutboundDone() && engine.isInboundDone();
+ }
+
+ private static void checkTransfer(ByteBuffer a, ByteBuffer b)
+ throws Exception {
+ a.flip();
+ b.flip();
+ assertTrue("transfer did not complete", a.equals(b));
+
+ a.position(a.limit());
+ b.position(b.limit());
+ a.limit(a.capacity());
+ b.limit(b.capacity());
+ }
+ @Test
+ public void testServerWeakCiphers() throws Exception {
+ // a simple test case to verify that SSL server rejects weak cipher suites,
+ // inspired by https://docs.oracle.com/javase/8/docs/technotes/guides/
+ // security/jsse/samples/sslengine/SSLEngineSimpleDemo.java
+
+ // set up a client and a server SSLEngine object, and let them exchange
+ // data over ByteBuffer instead of network socket.
+ GenericTestUtils.setLogLevel(SSLFactory.LOG, Level.DEBUG);
+ final Configuration conf = createConfiguration(true, true);
+
+ SSLFactory serverSSLFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
+ SSLFactory clientSSLFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+
+ serverSSLFactory.init();
+ clientSSLFactory.init();
+
+ SSLEngine serverSSLEngine = serverSSLFactory.createSSLEngine();
+ SSLEngine clientSSLEngine = clientSSLFactory.createSSLEngine();
+ // client selects cipher suites excluded by server
+ clientSSLEngine.setEnabledCipherSuites(excludeCiphers.split(","));
+
+ // use the same buffer size for server and client.
+ SSLSession session = clientSSLEngine.getSession();
+ int appBufferMax = session.getApplicationBufferSize();
+ int netBufferMax = session.getPacketBufferSize();
+
+ ByteBuffer clientOut = ByteBuffer.wrap("client".getBytes());
+ ByteBuffer clientIn = ByteBuffer.allocate(appBufferMax);
+ ByteBuffer serverOut = ByteBuffer.wrap("server".getBytes());
+ ByteBuffer serverIn = ByteBuffer.allocate(appBufferMax);
+
+ // send data from client to server
+ ByteBuffer cTOs = ByteBuffer.allocateDirect(netBufferMax);
+ // send data from server to client
+ ByteBuffer sTOc = ByteBuffer.allocateDirect(netBufferMax);
+
+ boolean dataDone = false;
+ try {
+ /**
+ * Server and client engines call wrap()/unwrap() to perform handshaking,
+ * until both engines are closed.
+ */
+ while (!isEngineClosed(clientSSLEngine) ||
+ !isEngineClosed(serverSSLEngine)) {
+ LOG.info("client wrap " + wrap(clientSSLEngine, clientOut, cTOs));
+ LOG.info("server wrap " + wrap(serverSSLEngine, serverOut, sTOc));
+ cTOs.flip();
+ sTOc.flip();
+ LOG.info("client unwrap " + unwrap(clientSSLEngine, sTOc, clientIn));
+ LOG.info("server unwrap " + unwrap(serverSSLEngine, cTOs, serverIn));
+ cTOs.compact();
+ sTOc.compact();
+ if (!dataDone && (clientOut.limit() == serverIn.position()) &&
+ (serverOut.limit() == clientIn.position())) {
+ checkTransfer(serverOut, clientIn);
+ checkTransfer(clientOut, serverIn);
+
+ LOG.info("closing client");
+ clientSSLEngine.closeOutbound();
+ dataDone = true;
+ }
+ }
+ Assert.fail("The exception was not thrown");
+ } catch (SSLHandshakeException e) {
+ GenericTestUtils.assertExceptionContains("no cipher suites in common", e);
+ }
+ }
+
+ private SSLEngineResult wrap(SSLEngine engine, ByteBuffer from,
+ ByteBuffer to) throws Exception {
+ SSLEngineResult result = engine.wrap(from, to);
+ runDelegatedTasks(result, engine);
+ return result;
+ }
+
+ private SSLEngineResult unwrap(SSLEngine engine, ByteBuffer from,
+ ByteBuffer to) throws Exception {
+ SSLEngineResult result = engine.unwrap(from, to);
+ runDelegatedTasks(result, engine);
+ return result;
+ }
+
@Test
public void validHostnameVerifier() throws Exception {
Configuration conf = createConfiguration(false, true);
[13/50] [abbrv] hadoop git commit: HADOOP-11393. Revert HADOOP_PREFIX,
go back to HADOOP_HOME (aw)
Posted by ae...@apache.org.
HADOOP-11393. Revert HADOOP_PREFIX, go back to HADOOP_HOME (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a74610d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a74610d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a74610d
Branch: refs/heads/HDFS-1312
Commit: 0a74610d1c7c7f183d2b2d0b7a775add53cf6c94
Parents: 0064cba
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Mar 24 08:47:00 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Mar 31 07:51:05 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/bin/hadoop | 12 +++---
.../hadoop-common/src/main/bin/hadoop-config.sh | 6 ++-
.../hadoop-common/src/main/bin/hadoop-daemon.sh | 6 +--
.../src/main/bin/hadoop-daemons.sh | 6 +--
.../src/main/bin/hadoop-functions.sh | 37 +++++++++---------
.../src/main/bin/hadoop-layout.sh.example | 14 +++----
.../hadoop-common/src/main/bin/slaves.sh | 6 +--
.../hadoop-common/src/main/bin/start-all.sh | 4 +-
.../hadoop-common/src/main/bin/stop-all.sh | 4 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 10 ++---
.../org/apache/hadoop/tracing/TraceUtils.java | 4 +-
.../src/site/markdown/ClusterSetup.md | 40 ++++++++++----------
.../src/site/markdown/CommandsManual.md | 2 +-
.../src/site/markdown/UnixShellGuide.md | 2 +-
.../scripts/hadoop-functions_test_helper.bash | 3 +-
.../src/test/scripts/hadoop_basic_init.bats | 2 +-
.../src/test/scripts/hadoop_bootstrap.bats | 4 +-
.../src/test/scripts/hadoop_confdir.bats | 24 ++++++------
.../src/test/scripts/hadoop_finalize.bats | 2 +-
.../hadoop-kms/src/main/conf/kms-env.sh | 4 +-
.../hadoop-kms/src/main/libexec/kms-config.sh | 8 ++--
.../hadoop-kms/src/main/sbin/kms.sh | 4 +-
.../src/main/conf/httpfs-env.sh | 4 +-
.../src/main/libexec/httpfs-config.sh | 8 ++--
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 4 +-
.../src/main/native/fuse-dfs/doc/README | 6 +--
.../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 12 +++---
.../src/main/bin/distribute-exclude.sh | 4 +-
.../hadoop-hdfs/src/main/bin/hdfs | 4 +-
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 6 +--
.../src/main/bin/refresh-namenodes.sh | 4 +-
.../hadoop-hdfs/src/main/bin/start-balancer.sh | 4 +-
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 4 +-
.../src/main/bin/start-secure-dns.sh | 4 +-
.../hadoop-hdfs/src/main/bin/stop-balancer.sh | 4 +-
.../hadoop-hdfs/src/main/bin/stop-dfs.sh | 4 +-
.../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 4 +-
.../hadoop-hdfs/src/site/markdown/Federation.md | 18 ++++-----
.../markdown/HDFSHighAvailabilityWithNFS.md | 4 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 4 +-
.../src/site/markdown/HdfsNfsGateway.md | 8 ++--
.../apache/hadoop/tracing/TestTraceAdmin.java | 2 +-
.../TestTracingShortCircuitLocalRead.java | 4 +-
hadoop-mapreduce-project/bin/mapred | 4 +-
hadoop-mapreduce-project/bin/mapred-config.sh | 6 +--
.../bin/mr-jobhistory-daemon.sh | 4 +-
.../apache/hadoop/mapred/pipes/Submitter.java | 2 +-
.../java/org/apache/hadoop/fs/DFSCIOTest.java | 2 +-
.../apache/hadoop/mapred/ReliabilityTest.java | 2 +-
.../apache/hadoop/tools/HadoopArchiveLogs.java | 4 +-
.../hadoop/tools/TestHadoopArchiveLogs.java | 4 +-
.../apache/hadoop/contrib/utils/join/README.txt | 2 +-
.../native/pipes/debug/pipes-default-script | 5 ++-
.../hadoop-sls/src/main/bin/rumen2sls.sh | 4 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 8 ++--
.../apache/hadoop/streaming/DumpTypedBytes.java | 2 +-
.../hadoop/streaming/HadoopStreaming.java | 2 +-
.../apache/hadoop/streaming/LoadTypedBytes.java | 2 +-
.../org/apache/hadoop/streaming/StreamJob.java | 16 ++++----
.../hadoop-yarn/bin/start-yarn.sh | 4 +-
.../hadoop-yarn/bin/stop-yarn.sh | 4 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +-
.../hadoop-yarn/bin/yarn-config.sh | 6 +--
.../hadoop-yarn/bin/yarn-daemon.sh | 4 +-
.../hadoop-yarn/bin/yarn-daemons.sh | 4 +-
.../TestDockerContainerExecutorWithMocks.java | 2 +-
.../site/markdown/DockerContainerExecutor.md.vm | 2 +-
67 files changed, 211 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 46eaf27..0756987 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -47,8 +47,8 @@ function hadoop_usage
# This script runs the hadoop core commands.
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
@@ -84,9 +84,9 @@ case ${COMMAND} in
# shellcheck disable=SC2086
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
- elif [[ -f "${HADOOP_PREFIX}/bin/hdfs" ]]; then
+ elif [[ -f "${HADOOP_HOME}/bin/hdfs" ]]; then
# shellcheck disable=SC2086
- exec "${HADOOP_PREFIX}/bin/hdfs" \
+ exec "${HADOOP_HOME}/bin/hdfs" \
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
else
hadoop_error "HADOOP_HDFS_HOME not found!"
@@ -104,8 +104,8 @@ case ${COMMAND} in
if [[ -f "${HADOOP_MAPRED_HOME}/bin/mapred" ]]; then
exec "${HADOOP_MAPRED_HOME}/bin/mapred" \
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
- elif [[ -f "${HADOOP_PREFIX}/bin/mapred" ]]; then
- exec "${HADOOP_PREFIX}/bin/mapred" \
+ elif [[ -f "${HADOOP_HOME}/bin/mapred" ]]; then
+ exec "${HADOOP_HOME}/bin/mapred" \
--config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
else
hadoop_error "HADOOP_MAPRED_HOME not found!"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index 0b52895..fd2c83e 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -63,6 +63,8 @@ else
exit 1
fi
+hadoop_deprecate_envvar HADOOP_PREFIX HADOOP_HOME
+
# allow overrides of the above and pre-defines of the below
if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
[[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh" ]]; then
@@ -128,8 +130,8 @@ fi
hadoop_shellprofiles_init
# get the native libs in there pretty quick
-hadoop_add_javalibpath "${HADOOP_PREFIX}/build/native"
-hadoop_add_javalibpath "${HADOOP_PREFIX}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
+hadoop_add_javalibpath "${HADOOP_HOME}/build/native"
+hadoop_add_javalibpath "${HADOOP_HOME}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
hadoop_shellprofiles_nativelib
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
index 5f094d6..8118f54 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemon.sh
@@ -21,8 +21,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
@@ -47,7 +47,7 @@ daemonmode=$1
shift
if [[ -z "${HADOOP_HDFS_HOME}" ]]; then
- hdfsscript="${HADOOP_PREFIX}/bin/hdfs"
+ hdfsscript="${HADOOP_HOME}/bin/hdfs"
else
hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
index 604eb7e..ae1e324 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
@@ -27,8 +27,8 @@ this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
@@ -51,7 +51,7 @@ daemonmode=$1
shift
if [[ -z "${HADOOP_HDFS_HOME}" ]]; then
- hdfsscript="${HADOOP_PREFIX}/bin/hdfs"
+ hdfsscript="${HADOOP_HOME}/bin/hdfs"
else
hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 7f293b6..6c4c345 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -278,7 +278,7 @@ function hadoop_bootstrap
# By now, HADOOP_LIBEXEC_DIR should have been defined upstream
# We can piggyback off of that to figure out where the default
# HADOOP_FREFIX should be. This allows us to run without
- # HADOOP_PREFIX ever being defined by a human! As a consequence
+ # HADOOP_HOME ever being defined by a human! As a consequence
# HADOOP_LIBEXEC_DIR now becomes perhaps the single most powerful
# env var within Hadoop.
if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
@@ -286,8 +286,8 @@ function hadoop_bootstrap
exit 1
fi
HADOOP_DEFAULT_PREFIX=$(cd -P -- "${HADOOP_LIBEXEC_DIR}/.." >/dev/null && pwd -P)
- HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX}
- export HADOOP_PREFIX
+ HADOOP_HOME=${HADOOP_HOME:-$HADOOP_DEFAULT_PREFIX}
+ export HADOOP_HOME
#
# short-cuts. vendors may redefine these as well, preferably
@@ -302,7 +302,7 @@ function hadoop_bootstrap
YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
- HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_PREFIX}}
+ HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}}
HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
@@ -326,12 +326,12 @@ function hadoop_find_confdir
# An attempt at compatibility with some Hadoop 1.x
# installs.
- if [[ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]]; then
+ if [[ -e "${HADOOP_HOME}/conf/hadoop-env.sh" ]]; then
conf_dir="conf"
else
conf_dir="etc/hadoop"
fi
- export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_PREFIX}/${conf_dir}}"
+ export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_HOME}/${conf_dir}}"
hadoop_debug "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}"
}
@@ -524,8 +524,8 @@ function hadoop_basic_init
hadoop_debug "Initialize CLASSPATH"
if [[ -z "${HADOOP_COMMON_HOME}" ]] &&
- [[ -d "${HADOOP_PREFIX}/${HADOOP_COMMON_DIR}" ]]; then
- export HADOOP_COMMON_HOME="${HADOOP_PREFIX}"
+ [[ -d "${HADOOP_HOME}/${HADOOP_COMMON_DIR}" ]]; then
+ export HADOOP_COMMON_HOME="${HADOOP_HOME}"
fi
# default policy file for service-level authorization
@@ -533,20 +533,20 @@ function hadoop_basic_init
# define HADOOP_HDFS_HOME
if [[ -z "${HADOOP_HDFS_HOME}" ]] &&
- [[ -d "${HADOOP_PREFIX}/${HDFS_DIR}" ]]; then
- export HADOOP_HDFS_HOME="${HADOOP_PREFIX}"
+ [[ -d "${HADOOP_HOME}/${HDFS_DIR}" ]]; then
+ export HADOOP_HDFS_HOME="${HADOOP_HOME}"
fi
# define HADOOP_YARN_HOME
if [[ -z "${HADOOP_YARN_HOME}" ]] &&
- [[ -d "${HADOOP_PREFIX}/${YARN_DIR}" ]]; then
- export HADOOP_YARN_HOME="${HADOOP_PREFIX}"
+ [[ -d "${HADOOP_HOME}/${YARN_DIR}" ]]; then
+ export HADOOP_YARN_HOME="${HADOOP_HOME}"
fi
# define HADOOP_MAPRED_HOME
if [[ -z "${HADOOP_MAPRED_HOME}" ]] &&
- [[ -d "${HADOOP_PREFIX}/${MAPRED_DIR}" ]]; then
- export HADOOP_MAPRED_HOME="${HADOOP_PREFIX}"
+ [[ -d "${HADOOP_HOME}/${MAPRED_DIR}" ]]; then
+ export HADOOP_MAPRED_HOME="${HADOOP_HOME}"
fi
if [[ ! -d "${HADOOP_COMMON_HOME}" ]]; then
@@ -573,7 +573,7 @@ function hadoop_basic_init
# let's define it as 'hadoop'
HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-$USER}
HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-hadoop}
- HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_PREFIX}/logs"}
+ HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_HOME}/logs"}
HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log}
HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO}
HADOOP_NICENESS=${HADOOP_NICENESS:-0}
@@ -1219,7 +1219,6 @@ function hadoop_finalize_hadoop_opts
hadoop_translate_cygwin_path HADOOP_LOG_DIR
hadoop_add_param HADOOP_OPTS hadoop.log.dir "-Dhadoop.log.dir=${HADOOP_LOG_DIR}"
hadoop_add_param HADOOP_OPTS hadoop.log.file "-Dhadoop.log.file=${HADOOP_LOGFILE}"
- HADOOP_HOME=${HADOOP_PREFIX}
hadoop_translate_cygwin_path HADOOP_HOME
export HADOOP_HOME
hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}"
@@ -1252,11 +1251,11 @@ function hadoop_finalize_catalina_opts
local prefix=${HADOOP_CATALINA_PREFIX}
- hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}"
+ hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}"
if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
hadoop_add_param CATALINA_OPTS java.library.path "-Djava.library.path=${JAVA_LIBRARY_PATH}"
fi
- hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_PREFIX}"
+ hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_HOME}"
hadoop_add_param CATALINA_OPTS "${prefix}.config.dir" "-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}"
hadoop_add_param CATALINA_OPTS "${prefix}.log.dir" "-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}"
hadoop_add_param CATALINA_OPTS "${prefix}.temp.dir" "-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}"
@@ -1282,7 +1281,7 @@ function hadoop_finalize
hadoop_finalize_hadoop_heap
hadoop_finalize_hadoop_opts
- hadoop_translate_cygwin_path HADOOP_PREFIX
+ hadoop_translate_cygwin_path HADOOP_HOME
hadoop_translate_cygwin_path HADOOP_CONF_DIR
hadoop_translate_cygwin_path HADOOP_COMMON_HOME
hadoop_translate_cygwin_path HADOOP_HDFS_HOME
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
index faa4317..efba10c 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
@@ -26,8 +26,8 @@
##
## If you move HADOOP_LIBEXEC_DIR from some location that
## isn't bin/../libexec, you MUST define either HADOOP_LIBEXEC_DIR
-## or have HADOOP_PREFIX/libexec/hadoop-config.sh and
-## HADOOP_PREFIX/libexec/hadoop-layout.sh (this file) exist.
+## or have HADOOP_HOME/libexec/hadoop-config.sh and
+## HADOOP_HOME/libexec/hadoop-layout.sh (this file) exist.
## NOTE:
##
@@ -44,7 +44,7 @@
####
# Default location for the common/core Hadoop project
-# export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
+# export HADOOP_COMMON_HOME=${HADOOP_HOME}
# Relative locations where components under HADOOP_COMMON_HOME are located
# export HADOOP_COMMON_DIR="share/hadoop/common"
@@ -56,7 +56,7 @@
####
# Default location for the HDFS subproject
-# export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
+# export HADOOP_HDFS_HOME=${HADOOP_HOME}
# Relative locations where components under HADOOP_HDFS_HOME are located
# export HDFS_DIR="share/hadoop/hdfs"
@@ -67,7 +67,7 @@
####
# Default location for the YARN subproject
-# export HADOOP_YARN_HOME=${HADOOP_PREFIX}
+# export HADOOP_YARN_HOME=${HADOOP_HOME}
# Relative locations where components under HADOOP_YARN_HOME are located
# export YARN_DIR="share/hadoop/yarn"
@@ -78,7 +78,7 @@
####
# Default location for the MapReduce subproject
-# export HADOOP_MAPRED_HOME=${HADOOP_PREFIX}
+# export HADOOP_MAPRED_HOME=${HADOOP_HOME}
# Relative locations where components under HADOOP_MAPRED_HOME are located
# export MAPRED_DIR="share/hadoop/mapreduce"
@@ -92,6 +92,6 @@
# note that this path only gets added for certain commands and not
# part of the general classpath unless HADOOP_OPTIONAL_TOOLS is used
# to configure them in
-# export HADOOP_TOOLS_HOME=${HADOOP_PREFIX}
+# export HADOOP_TOOLS_HOME=${HADOOP_HOME}
# export HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
# export HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh b/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
index 5859da0..34bf0eb 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
@@ -22,7 +22,7 @@
#
# HADOOP_SLAVES File naming remote hosts.
# Default is ${HADOOP_CONF_DIR}/slaves.
-# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf.
+# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf.
# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
# HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
##
@@ -33,8 +33,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh
index 845ca37..1420642 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh
@@ -21,8 +21,8 @@ exit 1
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
index df7ae8d..ee1f6eb 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
@@ -22,8 +22,8 @@ echo "This script is deprecated. Use stop-dfs.sh and stop-yarn.sh instead."
exit 1
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index 3c554aa..3f19e45 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -55,14 +55,14 @@
# Location of Hadoop. By default, Hadoop will attempt to determine
# this location based upon its execution path.
-# export HADOOP_PREFIX=
+# export HADOOP_HOME=
# Location of Hadoop's configuration information. i.e., where this
# file is probably living. Many sites will also set this in the
# same location where JAVA_HOME is defined. If this is not defined
# Hadoop will attempt to locate it based upon its execution
# path.
-# export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
+# export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
# The maximum amount of heap to use (Java -Xmx). If no unit
# is provided, it will be converted to MB. Daemons will
@@ -186,10 +186,10 @@ esac
# non-secure)
#
-# Where (primarily) daemon log files are stored. # $HADOOP_PREFIX/logs
-# by default.
+# Where (primarily) daemon log files are stored.
+# ${HADOOP_HOME}/logs by default.
# Java property: hadoop.log.dir
-# export HADOOP_LOG_DIR=${HADOOP_PREFIX}/logs
+# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
# A string representing this instance of hadoop. $USER by default.
# This is used in writing log and pid files, so keep that in mind!
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java
index 09acb35..0ae6d03 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java
@@ -32,7 +32,7 @@ import org.apache.htrace.core.HTraceConfiguration;
@InterfaceAudience.Private
public class TraceUtils {
private static List<ConfigurationPair> EMPTY = Collections.emptyList();
- static final String DEFAULT_HADOOP_PREFIX = "hadoop.htrace.";
+ static final String DEFAULT_HADOOP_TRACE_PREFIX = "hadoop.htrace.";
public static HTraceConfiguration wrapHadoopConf(final String prefix,
final Configuration conf) {
@@ -52,7 +52,7 @@ public class TraceUtils {
if (ret != null) {
return ret;
}
- return getInternal(DEFAULT_HADOOP_PREFIX + key);
+ return getInternal(DEFAULT_HADOOP_TRACE_PREFIX + key);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
index bf9419a..d2479e7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -86,10 +86,10 @@ Other useful configuration parameters that you can customize include:
In most cases, you should specify the `HADOOP_PID_DIR` and `HADOOP_LOG_DIR` directories such that they can only be written to by the users that are going to run the hadoop daemons. Otherwise there is the potential for a symlink attack.
-It is also traditional to configure `HADOOP_PREFIX` in the system-wide shell environment configuration. For example, a simple script inside `/etc/profile.d`:
+It is also traditional to configure `HADOOP_HOME` in the system-wide shell environment configuration. For example, a simple script inside `/etc/profile.d`:
- HADOOP_PREFIX=/path/to/hadoop
- export HADOOP_PREFIX
+ HADOOP_HOME=/path/to/hadoop
+ export HADOOP_HOME
| Daemon | Environment Variable |
|:---- |:---- |
@@ -243,73 +243,73 @@ To start a Hadoop cluster you will need to start both the HDFS and YARN cluster.
The first time you bring up HDFS, it must be formatted. Format a new distributed filesystem as *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format <cluster_name>
+ [hdfs]$ $HADOOP_HOME/bin/hdfs namenode -format <cluster_name>
Start the HDFS NameNode with the following command on the designated node as *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start namenode
Start a HDFS DataNode with the following command on each designated node as *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start datanode
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start datanode
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes can be started with a utility script. As *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh
+ [hdfs]$ $HADOOP_HOME/sbin/start-dfs.sh
Start the YARN with the following command, run on the designated ResourceManager as *yarn*:
- [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start resourcemanager
+ [yarn]$ $HADOOP_HOME/bin/yarn --daemon start resourcemanager
Run a script to start a NodeManager on each designated host as *yarn*:
- [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start nodemanager
+ [yarn]$ $HADOOP_HOME/bin/yarn --daemon start nodemanager
Start a standalone WebAppProxy server. Run on the WebAppProxy server as *yarn*. If multiple servers are used with load balancing it should be run on each of them:
- [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start proxyserver
+ [yarn]$ $HADOOP_HOME/bin/yarn --daemon start proxyserver
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be started with a utility script. As *yarn*:
- [yarn]$ $HADOOP_PREFIX/sbin/start-yarn.sh
+ [yarn]$ $HADOOP_HOME/sbin/start-yarn.sh
Start the MapReduce JobHistory Server with the following command, run on the designated server as *mapred*:
- [mapred]$ $HADOOP_PREFIX/bin/mapred --daemon start historyserver
+ [mapred]$ $HADOOP_HOME/bin/mapred --daemon start historyserver
### Hadoop Shutdown
Stop the NameNode with the following command, run on the designated NameNode as *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop namenode
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop namenode
Run a script to stop a DataNode as *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop datanode
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop datanode
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes may be stopped with a utility script. As *hdfs*:
- [hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh
+ [hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh
Stop the ResourceManager with the following command, run on the designated ResourceManager as *yarn*:
- [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop resourcemanager
+ [yarn]$ $HADOOP_HOME/bin/yarn --daemon stop resourcemanager
Run a script to stop a NodeManager on a slave as *yarn*:
- [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop nodemanager
+ [yarn]$ $HADOOP_HOME/bin/yarn --daemon stop nodemanager
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be stopped with a utility script. As *yarn*:
- [yarn]$ $HADOOP_PREFIX/sbin/stop-yarn.sh
+ [yarn]$ $HADOOP_HOME/sbin/stop-yarn.sh
Stop the WebAppProxy server. Run on the WebAppProxy server as *yarn*. If multiple servers are used with load balancing it should be run on each of them:
- [yarn]$ $HADOOP_PREFIX/bin/yarn stop proxyserver
+ [yarn]$ $HADOOP_HOME/bin/yarn stop proxyserver
Stop the MapReduce JobHistory Server with the following command, run on the designated server as *mapred*:
- [mapred]$ $HADOOP_PREFIX/bin/mapred --daemon stop historyserver
+ [mapred]$ $HADOOP_HOME/bin/mapred --daemon stop historyserver
Web Interfaces
--------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index 59ea198..365a844 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -39,7 +39,7 @@ All of the shell commands will accept a common set of options. For some commands
| SHELL\_OPTION | Description |
|:---- |:---- |
| `--buildpaths` | Enables developer versions of jars. |
-| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_PREFIX/etc/hadoop`. |
+| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_HOME/etc/hadoop`. |
| `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. For commands that do not support daemonization, this option is ignored. |
| `--debug` | Enables shell level configuration debugging information |
| `--help` | Shell script usage information. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
index ffab5a0..caa3aa7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
@@ -83,7 +83,7 @@ Apache Hadoop allows for third parties to easily add new features through a vari
Core to this functionality is the concept of a shell profile. Shell profiles are shell snippets that can do things such as add jars to the classpath, configure Java system properties and more.
-Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_PREFIX}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overriden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime.
+Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_HOME}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overriden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime.
An example of a shell profile is in the libexec directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
index f718345..be2d7f5 100755
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
@@ -27,6 +27,7 @@ setup() {
# shellcheck disable=SC2034
HADOOP_SHELL_SCRIPT_DEBUG=true
unset HADOOP_CONF_DIR
+ # we unset both of these for bw compat
unset HADOOP_HOME
unset HADOOP_PREFIX
@@ -53,4 +54,4 @@ strstr() {
else
echo false
fi
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats
index ae20248..79ede42 100644
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats
@@ -45,7 +45,7 @@ basicinitsetup () {
unset ${j}
done
- HADOOP_PREFIX=${TMP}
+ HADOOP_HOME=${TMP}
}
check_var_values () {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats
index 9114c70..de4edd4 100644
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats
@@ -22,7 +22,7 @@ load hadoop-functions_test_helper
}
@test "hadoop_bootstrap (libexec)" {
- unset HADOOP_PREFIX
+ unset HADOOP_HOME
unset HADOOP_COMMON_DIR
unset HADOOP_COMMON_LIB_JARS_DIR
unset HDFS_DIR
@@ -39,7 +39,7 @@ load hadoop-functions_test_helper
hadoop_bootstrap
# all of these should be set
- [ -n ${HADOOP_PREFIX} ]
+ [ -n ${HADOOP_HOME} ]
[ -n ${HADOOP_COMMON_DIR} ]
[ -n ${HADOOP_COMMON_LIB_JARS_DIR} ]
[ -n ${HDFS_DIR} ]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats
index 3e42da9..1f0c706 100644
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_confdir.bats
@@ -16,10 +16,10 @@
load hadoop-functions_test_helper
create_fake_dirs () {
- HADOOP_PREFIX=${TMP}
+ HADOOP_HOME=${TMP}
for j in conf etc/hadoop; do
- mkdir -p "${HADOOP_PREFIX}/${j}"
- echo "unittest=${j}" > "${HADOOP_PREFIX}/${j}/hadoop-env.sh"
+ mkdir -p "${HADOOP_HOME}/${j}"
+ echo "unittest=${j}" > "${HADOOP_HOME}/${j}/hadoop-env.sh"
done
}
@@ -32,27 +32,27 @@ create_fake_dirs () {
@test "hadoop_find_confdir (bw compat: conf)" {
create_fake_dirs
hadoop_find_confdir
- echo ">${HADOOP_CONF_DIR}< >${HADOOP_PREFIX}/conf<"
- [ "${HADOOP_CONF_DIR}" = ${HADOOP_PREFIX}/conf ]
+ echo ">${HADOOP_CONF_DIR}< >${HADOOP_HOME}/conf<"
+ [ "${HADOOP_CONF_DIR}" = ${HADOOP_HOME}/conf ]
}
@test "hadoop_find_confdir (etc/hadoop)" {
create_fake_dirs
- rm -rf "${HADOOP_PREFIX}/conf"
+ rm -rf "${HADOOP_HOME}/conf"
hadoop_find_confdir
- [ "${HADOOP_CONF_DIR}" = ${HADOOP_PREFIX}/etc/hadoop ]
+ [ "${HADOOP_CONF_DIR}" = ${HADOOP_HOME}/etc/hadoop ]
}
@test "hadoop_verify_confdir (negative) " {
create_fake_dirs
- HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf
+ HADOOP_CONF_DIR=${HADOOP_HOME}/conf
run hadoop_verify_confdir
[ -n "${output}" ]
}
@test "hadoop_verify_confdir (positive) " {
create_fake_dirs
- HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf
+ HADOOP_CONF_DIR=${HADOOP_HOME}/conf
touch "${HADOOP_CONF_DIR}/log4j.properties"
run hadoop_verify_confdir
[ -z "${output}" ]
@@ -60,7 +60,7 @@ create_fake_dirs () {
@test "hadoop_exec_hadoopenv (positive) " {
create_fake_dirs
- HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf
+ HADOOP_CONF_DIR=${HADOOP_HOME}/conf
hadoop_exec_hadoopenv
[ -n "${HADOOP_ENV_PROCESSED}" ]
[ "${unittest}" = conf ]
@@ -68,7 +68,7 @@ create_fake_dirs () {
@test "hadoop_exec_hadoopenv (negative) " {
create_fake_dirs
- HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf
+ HADOOP_CONF_DIR=${HADOOP_HOME}/conf
HADOOP_ENV_PROCESSED=true
hadoop_exec_hadoopenv
[ -z "${unittest}" ]
@@ -76,7 +76,7 @@ create_fake_dirs () {
@test "hadoop_exec_userfuncs" {
create_fake_dirs
- HADOOP_CONF_DIR=${HADOOP_PREFIX}/conf
+ HADOOP_CONF_DIR=${HADOOP_HOME}/conf
echo "unittest=userfunc" > "${HADOOP_CONF_DIR}/hadoop-user-functions.sh"
hadoop_exec_userfuncs
[ "${unittest}" = "userfunc" ]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats
index 668c115..b9339f3 100644
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_finalize.bats
@@ -100,7 +100,7 @@ load hadoop-functions_test_helper
hadoop_finalize_hadoop_heap () { true; }
hadoop_finalize_hadoop_opts () { true; }
hadoop_translate_cygwin_path () {
- if [ $1 = HADOOP_PREFIX ]; then
+ if [ $1 = HADOOP_HOME ]; then
testvar=prefix;
fi
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
index c3bc772..7044fa8 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
@@ -20,7 +20,7 @@
# KMS temporary directory
#
-# export KMS_TEMP=${HADOOP_PREFIX}/temp
+# export KMS_TEMP=${HADOOP_HOME}/temp
# The HTTP port used by KMS
#
@@ -59,7 +59,7 @@
#
# Location of tomcat
#
-# export KMS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/kms/tomcat
+# export KMS_CATALINA_HOME=${HADOOP_HOME}/share/hadoop/kms/tomcat
# Java System properties for KMS should be specified in this variable.
# The java.library.path and hadoop.home.dir properties are automatically
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
index c88aa87..5e1ffa4 100644
--- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
@@ -28,7 +28,7 @@ function hadoop_subproject_init
export HADOOP_CATALINA_PREFIX=kms
- export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_PREFIX}/temp}"
+ export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_HOME}/temp}"
hadoop_deprecate_envvar KMS_CONFIG HADOOP_CONF_DIR
@@ -49,7 +49,7 @@ function hadoop_subproject_init
# shellcheck disable=SC2086
export KMS_SSL_TRUSTSTORE_PASS=${KMS_SSL_TRUSTSTORE_PASS:-"$(echo ${CATALINA_OPTS} | grep -o 'trustStorePassword=[^ ]*' | cut -f2 -d= )"}
- export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/kms/tomcat}"
+ export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_HOME}/share/hadoop/kms/tomcat}"
export HADOOP_CATALINA_HOME="${KMS_CATALINA_HOME:-${CATALINA_BASE}}"
export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out}"
@@ -69,8 +69,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then
- . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
else
echo "ERROR: Hadoop common not found." 2>&1
exit 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
index 4ed4725..6708cd9 100755
--- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
@@ -30,8 +30,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
index a4edef6..f012453 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
@@ -20,7 +20,7 @@
# HTTPFS temporary directory
#
-# export HTTPFS_TEMP=${HADOOP_PREFIX}/temp
+# export HTTPFS_TEMP=${HADOOP_HOME}/temp
# The HTTP port used by HTTPFS
#
@@ -53,7 +53,7 @@
#
# Location of tomcat
#
-# export HTTPFS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat
+# export HTTPFS_CATALINA_HOME=${HADOOP_HOME}/share/hadoop/httpfs/tomcat
# Java System properties for HTTPFS should be specified in this variable.
# The java.library.path and hadoop.home.dir properties are automatically
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
index 767bd6e..ba4b406 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
@@ -28,7 +28,7 @@ function hadoop_subproject_init
export HADOOP_CATALINA_PREFIX=httpfs
- export HADOOP_CATALINA_TEMP="${HTTPFS_TEMP:-${HADOOP_PREFIX}/temp}"
+ export HADOOP_CATALINA_TEMP="${HTTPFS_TEMP:-${HADOOP_HOME}/temp}"
hadoop_deprecate_envvar HTTPFS_CONFIG HADOOP_CONF_DIR
@@ -47,7 +47,7 @@ function hadoop_subproject_init
export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${HTTPFS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}"
- export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat}"
+ export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_HOME}/share/hadoop/httpfs/tomcat}"
export HADOOP_CATALINA_HOME="${HTTPFS_CATALINA_HOME:-${CATALINA_BASE}}"
export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs-${HOSTNAME}.out}"
@@ -67,8 +67,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then
- . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
else
echo "ERROR: Hadoop common not found." 2>&1
exit 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index 18c1af0..3e7cdf8 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -30,8 +30,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
index 672265e..e8cc0e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
@@ -35,9 +35,9 @@ Requirements
BUILDING
fuse-dfs executable can be built by setting `require.fuse` option to true using Maven. For example:
- in HADOOP_PREFIX: `mvn package -Pnative -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true`
+ in HADOOP_HOME: `mvn package -Pnative -Drequire.fuse=true -DskipTests -Dmaven.javadoc.skip=true`
- The executable `fuse_dfs` will be located at HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/
+ The executable `fuse_dfs` will be located at HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/
Common build problems include not finding the libjvm.so in JAVA_HOME/jre/lib/OS_ARCH/server or not finding fuse in FUSE_HOME or /usr/local.
@@ -109,7 +109,7 @@ NOTE - you cannot export this with a FUSE module built into the kernel
RECOMMENDATIONS
-1. From /bin, `ln -s HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs* .`
+1. From /bin, `ln -s HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs* .`
2. Always start with debug on so you can see if you are missing a classpath or something like that.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh
index 26dfd19..c52c5f9 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs_wrapper.sh
@@ -16,12 +16,12 @@
# limitations under the License.
#
-if [ "$HADOOP_PREFIX" = "" ]; then
- echo "HADOOP_PREFIX is empty. Set it to the root directory of Hadoop source code"
+if [ "$HADOOP_HOME" = "" ]; then
+ echo "HADOOP_HOME is empty. Set it to the root directory of Hadoop source code"
exit 1
fi
-export FUSEDFS_PATH="$HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs"
-export LIBHDFS_PATH="$HADOOP_PREFIX/hadoop-hdfs-project/hadoop-hdfs-native-client/target/usr/local/lib"
+export FUSEDFS_PATH="$HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs"
+export LIBHDFS_PATH="$HADOOP_HOME/hadoop-hdfs-project/hadoop-hdfs-native-client/target/usr/local/lib"
if [ "$OS_ARCH" = "" ]; then
export OS_ARCH=amd64
@@ -38,12 +38,12 @@ fi
while IFS= read -r -d '' file
do
export CLASSPATH=$CLASSPATH:$file
-done < <(find "$HADOOP_PREFIX/hadoop-client" -name "*.jar" -print0)
+done < <(find "$HADOOP_HOME/hadoop-client" -name "*.jar" -print0)
while IFS= read -r -d '' file
do
export CLASSPATH=$CLASSPATH:$file
-done < <(find "$HADOOP_PREFIX/hhadoop-hdfs-project" -name "*.jar" -print0)
+done < <(find "$HADOOP_HOME/hhadoop-hdfs-project" -name "*.jar" -print0)
export CLASSPATH=$HADOOP_CONF_DIR:$CLASSPATH
export PATH=$FUSEDFS_PATH:$PATH
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh
index cfd44e3..97f04f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh
@@ -52,8 +52,8 @@ if [ ! -f "$excludeFilenameLocal" ] ; then
exit 1
fi
-namenodes=$("$HADOOP_PREFIX/bin/hdfs" getconf -namenodes)
-excludeFilenameRemote=$("$HADOOP_PREFIX/bin/hdfs" getconf -excludeFile)
+namenodes=$("$HADOOP_HOME/bin/hdfs" getconf -namenodes)
+excludeFilenameRemote=$("$HADOOP_HOME/bin/hdfs" getconf -excludeFile)
if [ "$excludeFilenameRemote" = '' ] ; then
echo \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index bcd04d1..c365250 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -60,8 +60,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
index 244e5a9..d440210 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
@@ -49,7 +49,7 @@ function hadoop_subproject_init
hadoop_deprecate_envvar HADOOP_HDFS_IDENT_STRING HADOOP_IDENT_STRING
- HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_PREFIX}"
+ HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME}"
# turn on the defaults
export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-INFO,NullAppender}
@@ -71,8 +71,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then
- . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
+elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
else
echo "ERROR: Hadoop common not found." 2>&1
exit 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh
index 318a282..f51dd0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh
@@ -21,8 +21,8 @@
# for dfsadmin to support multiple namenodes.
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
index 32ca2b2..df044fe 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
@@ -30,8 +30,8 @@ function hadoop_usage
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
index 9c5a172..1e35e7d 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
@@ -29,8 +29,8 @@ this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh
index f904640..3fce345 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh
@@ -26,8 +26,8 @@ this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
index bb51a8a..ec94080 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
@@ -28,8 +28,8 @@ function hadoop_usage
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
index cc0d11d..e693374 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
@@ -28,8 +28,8 @@ this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh
index 816a3e3..2a973b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh
@@ -26,8 +26,8 @@ this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
index 38c1070..99a41a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
@@ -150,13 +150,13 @@ Here is an example configuration with two Namenodes:
**Step 1**: Format a Namenode using the following command:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format [-clusterId <cluster_id>]
+ [hdfs]$ $HADOOP_HOME/bin/hdfs namenode -format [-clusterId <cluster_id>]
Choose a unique cluster\_id which will not conflict other clusters in your environment. If a cluster\_id is not provided, then a unique one is auto generated.
**Step 2**: Format additional Namenodes using the following command:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format -clusterId <cluster_id>
+ [hdfs]$ $HADOOP_HOME/bin/hdfs namenode -format -clusterId <cluster_id>
Note that the cluster\_id in step 2 must be same as that of the cluster\_id in step 1. If they are different, the additional Namenodes will not be part of the federated cluster.
@@ -164,7 +164,7 @@ Note that the cluster\_id in step 2 must be same as that of the cluster\_id in s
Older releases only support a single Namenode. Upgrade the cluster to newer release in order to enable federation During upgrade you can provide a ClusterID as follows:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode -upgrade -clusterId <cluster_ID>
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start namenode -upgrade -clusterId <cluster_ID>
If cluster\_id is not provided, it is auto generated.
@@ -187,7 +187,7 @@ Perform the following steps:
* Refresh the Datanodes to pickup the newly added Namenode by running
the following command against all the Datanodes in the cluster:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNamenodes <datanode_host_name>:<datanode_rpc_port>
+ [hdfs]$ $HADOOP_HOME/bin/hdfs dfsadmin -refreshNamenodes <datanode_host_name>:<datanode_rpc_port>
Managing the cluster
--------------------
@@ -196,11 +196,11 @@ Managing the cluster
To start the cluster run the following command:
- [hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh
+ [hdfs]$ $HADOOP_HOME/sbin/start-dfs.sh
To stop the cluster run the following command:
- [hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh
+ [hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh
These commands can be run from any node where the HDFS configuration is available. The command uses the configuration to determine the Namenodes in the cluster and then starts the Namenode process on those nodes. The Datanodes are started on the nodes specified in the `slaves` file. The script can be used as a reference for building your own scripts to start and stop the cluster.
@@ -208,7 +208,7 @@ These commands can be run from any node where the HDFS configuration is availabl
The Balancer has been changed to work with multiple Namenodes. The Balancer can be run using the command:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start balancer [-policy <policy>]
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start balancer [-policy <policy>]
The policy parameter can be any of the following:
@@ -227,11 +227,11 @@ Decommissioning is similar to prior releases. The nodes that need to be decomiss
**Step 1**: To distribute an exclude file to all the Namenodes, use the following command:
- [hdfs]$ $HADOOP_PREFIX/sbin/distribute-exclude.sh <exclude_file>
+ [hdfs]$ $HADOOP_HOME/sbin/distribute-exclude.sh <exclude_file>
**Step 2**: Refresh all the Namenodes to pick up the new exclude file:
- [hdfs]$ $HADOOP_PREFIX/sbin/refresh-namenodes.sh
+ [hdfs]$ $HADOOP_HOME/sbin/refresh-namenodes.sh
The above command uses HDFS configuration to determine the configured Namenodes in the cluster and refreshes them to pick up the new exclude file.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
index 51a88c9..f888966 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
@@ -475,7 +475,7 @@ There are also several other configuration parameters which may be set to contro
After the configuration keys have been added, the next step is to initialize required state in ZooKeeper. You can do so by running the following command from one of the NameNode hosts.
- [hdfs]$ $HADOOP_PREFIX/bin/zkfc -formatZK
+ [hdfs]$ $HADOOP_HOME/bin/zkfc -formatZK
This will create a znode in ZooKeeper inside of which the automatic failover system stores its data.
@@ -487,7 +487,7 @@ Since automatic failover has been enabled in the configuration, the `start-dfs.s
If you manually manage the services on your cluster, you will need to manually start the `zkfc` daemon on each of the machines that runs a NameNode. You can start the daemon by running:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start zkfc
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start zkfc
### Securing access to ZooKeeper
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index 8b42386..9a97add 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -523,7 +523,7 @@ There are also several other configuration parameters which may be set to contro
After the configuration keys have been added, the next step is to initialize required state in ZooKeeper. You can do so by running the following command from one of the NameNode hosts.
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs zkfc -formatZK
+ [hdfs]$ $HADOOP_HOME/bin/hdfs zkfc -formatZK
This will create a znode in ZooKeeper inside of which the automatic failover system stores its data.
@@ -535,7 +535,7 @@ Since automatic failover has been enabled in the configuration, the `start-dfs.s
If you manually manage the services on your cluster, you will need to manually start the `zkfc` daemon on each of the machines that runs a NameNode. You can start the daemon by running:
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start zkfc
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start zkfc
### Securing access to ZooKeeper
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
index 7dc2fe4..6731189 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
@@ -215,7 +215,7 @@ Three daemons are required to provide NFS service: rpcbind (or portmap), mountd
2. Start Hadoop's portmap (needs root privileges):
- [root]> $HADOOP_PREFIX/bin/hdfs --daemon start portmap
+ [root]> $HADOOP_HOME/bin/hdfs --daemon start portmap
3. Start mountd and nfsd.
@@ -224,12 +224,12 @@ Three daemons are required to provide NFS service: rpcbind (or portmap), mountd
While in secure mode, any user can start NFS gateway
as long as the user has read access to the Kerberos keytab defined in "nfs.keytab.file".
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start nfs3
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start nfs3
4. Stop NFS gateway services.
- [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop nfs3
- [root]> $HADOOP_PREFIX/bin/hdfs --daemon stop portmap
+ [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop nfs3
+ [root]> $HADOOP_HOME/bin/hdfs --daemon stop portmap
Optionally, you can forgo running the Hadoop-provided portmap daemon and instead use the system portmap daemon on all operating systems if you start the NFS Gateway as root. This will allow the HDFS NFS Gateway to work around the aforementioned bug and still register using the system portmap daemon. To do so, just start the NFS gateway daemon as you normally would, but make sure to do so as the "root" user, and also set the "HADOOP\_PRIVILEGED\_NFS\_USER" environment variable to an unprivileged user. In this mode the NFS Gateway will start as root to perform its initial registration with the system portmap, and then will drop privileges back to the user specified by the HADOOP\_PRIVILEGED\_NFS\_USER afterward and for the rest of the duration of the lifetime of the NFS Gateway process. Note that if you choose this route, you should skip steps 1 and 2 above.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
index 198dafb..71c9c56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
@@ -62,7 +62,7 @@ public class TestTraceAdmin {
public void testCreateAndDestroySpanReceiver() throws Exception {
Configuration conf = new Configuration();
conf = new Configuration();
- conf.set(TraceUtils.DEFAULT_HADOOP_PREFIX +
+ conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX +
Tracer.SPAN_RECEIVER_CLASSES_KEY, "");
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
index 37c09d1..b3cf402 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
@@ -65,10 +65,10 @@ public class TestTracingShortCircuitLocalRead {
public void testShortCircuitTraceHooks() throws IOException {
assumeTrue(NativeCodeLoader.isNativeCodeLoaded() && !Path.WINDOWS);
conf = new Configuration();
- conf.set(TraceUtils.DEFAULT_HADOOP_PREFIX +
+ conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX +
Tracer.SPAN_RECEIVER_CLASSES_KEY,
SetSpanReceiver.class.getName());
- conf.set(TraceUtils.DEFAULT_HADOOP_PREFIX +
+ conf.set(TraceUtils.DEFAULT_HADOOP_TRACE_PREFIX +
Tracer.SAMPLER_CLASSES_KEY,
"AlwaysSampler");
conf.setLong("dfs.blocksize", 100 * 1024);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index fab5b87..f280f31 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -37,8 +37,8 @@ function hadoop_usage
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-mapreduce-project/bin/mapred-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh
index fe6e9e6..a989792 100644
--- a/hadoop-mapreduce-project/bin/mapred-config.sh
+++ b/hadoop-mapreduce-project/bin/mapred-config.sh
@@ -47,7 +47,7 @@ function hadoop_subproject_init
hadoop_deprecate_envvar HADOOP_MAPRED_ROOT_LOGGER HADOOP_ROOT_LOGGER
- HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_PREFIX}"
+ HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_HOME}"
hadoop_deprecate_envvar HADOOP_MAPRED_IDENT_STRING HADOOP_IDENT_STRING
}
@@ -62,8 +62,8 @@ if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then
- . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
+elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_HOME}/libexec/hadoop-config.sh"
else
echo "ERROR: Hadoop common not found." 2>&1
exit 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh b/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
index 57b1ebd..998ca90 100644
--- a/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
+++ b/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
@@ -21,8 +21,8 @@ function hadoop_usage
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
index 4f5b6a1..ae45782 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
@@ -315,7 +315,7 @@ public class Submitter extends Configured implements Tool {
// <path>#<executable>
if (exec.contains("#")) {
// set default gdb commands for map and reduce task
- String defScript = "$HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-script";
+ String defScript = "$HADOOP_HOME/src/c++/pipes/debug/pipes-default-script";
setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript);
setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
index 12bec08..b01954e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
@@ -450,7 +450,7 @@ public class DFSCIOTest {
}
//Copy the executables over to the remote filesystem
- String hadoopHome = System.getenv("HADOOP_PREFIX");
+ String hadoopHome = System.getenv("HADOOP_HOME");
fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/libhdfs.so." + HDFS_LIB_VERSION),
HDFS_SHLIB);
fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/hdfs_read"), HDFS_READ);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
index e6e12eb..ecac83a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
@@ -179,7 +179,7 @@ public class ReliabilityTest extends Configured implements Tool {
private String normalizeCommandPath(String command) {
final String hadoopHome;
- if ((hadoopHome = System.getenv("HADOOP_PREFIX")) != null) {
+ if ((hadoopHome = System.getenv("HADOOP_HOME")) != null) {
command = hadoopHome + "/" + command;
}
return command;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java
index 6b8af97..c502ffd 100644
--- a/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java
+++ b/hadoop-tools/hadoop-archive-logs/src/main/java/org/apache/hadoop/tools/HadoopArchiveLogs.java
@@ -450,7 +450,7 @@ public class HadoopArchiveLogs implements Tool {
fi
export HADOOP_CLIENT_OPTS="-Xmx1024m"
export HADOOP_CLASSPATH=/dist/share/hadoop/tools/lib/hadoop-archive-logs-2.8.0-SNAPSHOT.jar:/dist/share/hadoop/tools/lib/hadoop-archives-2.8.0-SNAPSHOT.jar
- "$HADOOP_PREFIX"/bin/hadoop org.apache.hadoop.tools.HadoopArchiveLogsRunner -appId "$appId" -user "$user" -workingDir /tmp/logs/archive-logs-work -remoteRootLogDir /tmp/logs -suffix logs
+ "$HADOOP_HOME"/bin/hadoop org.apache.hadoop.tools.HadoopArchiveLogsRunner -appId "$appId" -user "$user" -workingDir /tmp/logs/archive-logs-work -remoteRootLogDir /tmp/logs -suffix logs
*/
@VisibleForTesting
void generateScript(File localScript, Path workingDir,
@@ -484,7 +484,7 @@ public class HadoopArchiveLogs implements Tool {
fw.write("m\"\n");
fw.write("export HADOOP_CLASSPATH=");
fw.write(classpath);
- fw.write("\n\"$HADOOP_PREFIX\"/bin/hadoop ");
+ fw.write("\n\"$HADOOP_HOME\"/bin/hadoop ");
fw.write(HadoopArchiveLogsRunner.class.getName());
fw.write(" -appId \"$appId\" -user \"$user\" -workingDir ");
fw.write(workingDir.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
index 7fcb0bf..d2d7801 100644
--- a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
+++ b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
@@ -294,14 +294,14 @@ public class TestHadoopArchiveLogs {
Assert.assertTrue(lines[14].startsWith("export HADOOP_CLASSPATH="));
if (proxy) {
Assert.assertEquals(
- "\"$HADOOP_PREFIX\"/bin/hadoop org.apache.hadoop.tools." +
+ "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." +
"HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " +
"-workingDir " + workingDir.toString() + " -remoteRootLogDir " +
remoteRootLogDir.toString() + " -suffix " + suffix,
lines[15]);
} else {
Assert.assertEquals(
- "\"$HADOOP_PREFIX\"/bin/hadoop org.apache.hadoop.tools." +
+ "\"$HADOOP_HOME\"/bin/hadoop org.apache.hadoop.tools." +
"HadoopArchiveLogsRunner -appId \"$appId\" -user \"$user\" " +
"-workingDir " + workingDir.toString() + " -remoteRootLogDir " +
remoteRootLogDir.toString() + " -suffix " + suffix + " -noProxy",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt
index 73fd6ef..47ef31c 100644
--- a/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt
+++ b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/README.txt
@@ -20,7 +20,7 @@ B.a31 B.a32
*****************************
*** Invoke SampleDataJoin ***
*****************************
-[:~]$ $HADOOP_PREFIX/bin/hadoop jar hadoop-datajoin-examples.jar org.apache.hadoop.contrib.utils.join.DataJoinJob datajoin/input datajoin/output Text 1 org.apache.hadoop.contrib.utils.join.SampleDataJoinMapper org.apache.hadoop.contrib.utils.join.SampleDataJoinReducer org.apache.hadoop.contrib.utils.join.SampleTaggedMapOutput Text
+[:~]$ $HADOOP_HOME/bin/hadoop jar hadoop-datajoin-examples.jar org.apache.hadoop.contrib.utils.join.DataJoinJob datajoin/input datajoin/output Text 1 org.apache.hadoop.contrib.utils.join.SampleDataJoinMapper org.apache.hadoop.contrib.utils.join.SampleDataJoinReducer org.apache.hadoop.contrib.utils.join.SampleTaggedMapOutput Text
Using TextInputFormat: Text
Using TextOutputFormat: Text
07/06/01 19:58:23 INFO mapred.FileInputFormat: Total input paths to process : 2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script
index 6bacc43..7b74fb6 100644
--- a/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script
+++ b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -9,6 +10,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-core=`find . -name 'core*'`
+core=$(find . -name 'core*')
#Only pipes programs have 5th argument as program name.
-gdb -quiet $5 -c $core -x $HADOOP_PREFIX/src/c++/pipes/debug/pipes-default-gdb-commands.txt
+gdb -quiet "${5}" -c "${core}" -x "${HADOOP_HOME}/src/c++/pipes/debug/pipes-default-gdb-commands.txt"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
index f9bfaef..0bd291b 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
@@ -77,8 +77,8 @@ function run_sls_generator()
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index 30fd60a..403c4bb 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -71,8 +71,8 @@ function parse_args()
function calculate_classpath
{
hadoop_add_to_classpath_tools hadoop-sls
- hadoop_debug "Injecting ${HADOOP_PREFIX}/share/hadoop/tools/sls/html into CLASSPATH"
- hadoop_add_classpath "${HADOOP_PREFIX}/share/hadoop/tools/sls/html"
+ hadoop_debug "Injecting ${HADOOP_TOOLS_DIR}/sls/html into CLASSPATH"
+ hadoop_add_classpath "${HADOOP_TOOLS_DIR}/sls/html"
}
function run_simulation() {
@@ -105,8 +105,8 @@ function run_simulation() {
}
# let's locate libexec...
-if [[ -n "${HADOOP_PREFIX}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+if [[ -n "${HADOOP_HOME}" ]]; then
+ HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
this="${BASH_SOURCE-$0}"
bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
index 42007a07..5a07cc3 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
@@ -91,7 +91,7 @@ public class DumpTypedBytes implements Tool {
}
private void printUsage() {
- System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar"
+ System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
+ " dumptb <glob-pattern>");
System.out.println(" Dumps all files that match the given pattern to " +
"standard output as typed bytes.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a74610d/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
index 5d01124..eabf46c 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
@@ -56,7 +56,7 @@ public class HadoopStreaming {
}
private static void printUsage() {
- System.out.println("Usage: $HADOOP_PREFIX/bin/hadoop jar hadoop-streaming.jar"
+ System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
+ " [options]");
System.out.println("Options:");
System.out.println(" dumptb <glob-pattern> Dumps all files that match the"
[19/50] [abbrv] hadoop git commit: YARN-4634. Scheduler UI/Metrics
need to consider cases like non-queue label mappings. (Sunil G via wangda)
Posted by ae...@apache.org.
YARN-4634. Scheduler UI/Metrics need to consider cases like non-queue label mappings. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12b11e2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12b11e2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12b11e2e
Branch: refs/heads/HDFS-1312
Commit: 12b11e2e688158404feeb3ded37eb6cccad4ea5c
Parents: 0dd9bca
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Mar 31 14:32:37 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Mar 31 14:35:18 2016 -0700
----------------------------------------------------------------------
.../webapp/CapacitySchedulerPage.java | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12b11e2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 033152a..5abc250 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -405,8 +405,20 @@ class CapacitySchedulerPage extends RmView {
CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root, cs);
csqinfo.csinfo = sinfo;
- if (null == nodeLabelsInfo || (nodeLabelsInfo.size() == 1
- && nodeLabelsInfo.get(0).getLabelName().isEmpty())) {
+ boolean hasAnyLabelLinkedToNM = false;
+ if (null != nodeLabelsInfo) {
+ for (RMNodeLabel label : nodeLabelsInfo) {
+ if (label.getLabelName().length() == 0) {
+ // Skip DEFAULT_LABEL
+ continue;
+ }
+ if (label.getNumActiveNMs() > 0) {
+ hasAnyLabelLinkedToNM = true;
+ break;
+ }
+ }
+ }
+ if (!hasAnyLabelLinkedToNM) {
used = sinfo.getUsedCapacity() / 100;
//label is not enabled in the cluster or there's only "default" label,
ul.li().
[06/50] [abbrv] hadoop git commit: HDFS-10223. peerFromSocketAndKey
performs SASL exchange before setting connection timeouts (cmccabe)
Posted by ae...@apache.org.
HDFS-10223. peerFromSocketAndKey performs SASL exchange before setting connection timeouts (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37e23ce4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37e23ce4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37e23ce4
Branch: refs/heads/HDFS-1312
Commit: 37e23ce45c592f3c9c48a08a52a5f46787f6c0e9
Parents: 60e4116
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Mar 30 13:37:37 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Mar 30 13:37:37 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 6 ++-
.../erasurecode/ErasureCodingWorker.java | 3 +-
.../hdfs/server/namenode/NamenodeFsck.java | 2 +-
.../datatransfer/sasl/TestSaslDataTransfer.java | 48 ++++++++++++++++++++
5 files changed, 55 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37e23ce4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 88bd219..d4e3187 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2734,9 +2734,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
NetUtils.connect(sock, addr, getRandomLocalInterfaceAddr(),
socketTimeout);
peer = DFSUtilClient.peerFromSocketAndKey(saslClient, sock, this,
- blockToken, datanodeId);
- peer.setReadTimeout(socketTimeout);
- peer.setWriteTimeout(socketTimeout);
+ blockToken, datanodeId, socketTimeout);
success = true;
return peer;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37e23ce4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index 880234e..b9f4dce2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -587,12 +587,14 @@ public class DFSUtilClient {
public static Peer peerFromSocketAndKey(
SaslDataTransferClient saslClient, Socket s,
DataEncryptionKeyFactory keyFactory,
- Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
- throws IOException {
+ Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId,
+ int socketTimeoutMs) throws IOException {
Peer peer = null;
boolean success = false;
try {
peer = peerFromSocket(s);
+ peer.setReadTimeout(socketTimeoutMs);
+ peer.setWriteTimeout(socketTimeoutMs);
peer = saslClient.peerSend(peer, keyFactory, blockToken, datanodeId);
success = true;
return peer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37e23ce4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 74fb3e1..4bcb291 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -875,8 +875,7 @@ public final class ErasureCodingWorker {
NetUtils.connect(sock, addr, socketTimeout);
peer = DFSUtilClient.peerFromSocketAndKey(datanode.getSaslClient(),
sock, datanode.getDataEncryptionKeyFactoryForBlock(b),
- blockToken, datanodeId);
- peer.setReadTimeout(socketTimeout);
+ blockToken, datanodeId, socketTimeout);
success = true;
return peer;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37e23ce4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index d3be9b4..291ba56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -952,7 +952,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
s.setSoTimeout(HdfsConstants.READ_TIMEOUT);
peer = DFSUtilClient.peerFromSocketAndKey(
dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
- blockToken, datanodeId);
+ blockToken, datanodeId, HdfsConstants.READ_TIMEOUT);
} finally {
if (peer == null) {
IOUtils.closeQuietly(s);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37e23ce4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
index 2d4eb0d..8555e5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
@@ -25,6 +25,10 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.BlockLocation;
@@ -32,12 +36,18 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.junit.After;
@@ -197,4 +207,42 @@ public class TestSaslDataTransfer extends SaslDataTransferTestCase {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
}
+
+ /**
+ * Verifies that peerFromSocketAndKey honors socket read timeouts.
+ */
+ @Test(timeout=60000)
+ public void TestPeerFromSocketAndKeyReadTimeout() throws Exception {
+ HdfsConfiguration conf = createSecureConfig(
+ "authentication,integrity,privacy");
+ AtomicBoolean fallbackToSimpleAuth = new AtomicBoolean(false);
+ SaslDataTransferClient saslClient = new SaslDataTransferClient(
+ conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf),
+ TrustedChannelResolver.getInstance(conf), fallbackToSimpleAuth);
+ DatanodeID fakeDatanodeId = new DatanodeID("127.0.0.1", "localhost",
+ "beefbeef-beef-beef-beef-beefbeefbeef", 1, 2, 3, 4);
+ DataEncryptionKeyFactory dataEncKeyFactory =
+ new DataEncryptionKeyFactory() {
+ @Override
+ public DataEncryptionKey newDataEncryptionKey() {
+ return new DataEncryptionKey(123, "456", new byte[8],
+ new byte[8], 1234567, "fakeAlgorithm");
+ }
+ };
+ ServerSocket serverSocket = null;
+ Socket socket = null;
+ try {
+ serverSocket = new ServerSocket(0, -1);
+ socket = new Socket(serverSocket.getInetAddress(),
+ serverSocket.getLocalPort());
+ Peer peer = DFSUtilClient.peerFromSocketAndKey(saslClient, socket,
+ dataEncKeyFactory, new Token(), fakeDatanodeId, 1);
+ peer.close();
+ Assert.fail("Expected DFSClient#peerFromSocketAndKey to time out.");
+ } catch (SocketTimeoutException e) {
+ GenericTestUtils.assertExceptionContains("Read timed out", e);
+ } finally {
+ IOUtils.cleanup(null, socket, serverSocket);
+ }
+ }
}
[49/50] [abbrv] hadoop git commit: YARN-4917. Fix typos in
documentation of Capacity Scheduler. (Takashi Ohnishi via iwasakims)
Posted by ae...@apache.org.
YARN-4917. Fix typos in documentation of Capacity Scheduler. (Takashi Ohnishi via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/500e5a59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/500e5a59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/500e5a59
Branch: refs/heads/HDFS-1312
Commit: 500e5a5952f8f34bf0e1e2653fa01b357d68cc8f
Parents: 3020634
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Apr 6 04:00:31 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Apr 6 04:00:31 2016 +0900
----------------------------------------------------------------------
.../src/site/markdown/CapacityScheduler.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/500e5a59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index e86c4f9..8c0b8c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -55,11 +55,11 @@ The `CapacityScheduler` supports the following features:
* **Hierarchical Queues** - Hierarchy of queues is supported to ensure resources are shared among the sub-queues of an organization before other queues are allowed to use free resources, there-by providing more control and predictability.
-* **Capacity Guarantees** - Queues are allocated a fraction of the capacity of the grid in the sense that a certain capacity of resources will be at their disposal. All applications submitted to a queue will have access to the capacity allocated to the queue. Adminstrators can configure soft limits and optional hard limits on the capacity allocated to each queue.
+* **Capacity Guarantees** - Queues are allocated a fraction of the capacity of the grid in the sense that a certain capacity of resources will be at their disposal. All applications submitted to a queue will have access to the capacity allocated to the queue. Administrators can configure soft limits and optional hard limits on the capacity allocated to each queue.
* **Security** - Each queue has strict ACLs which controls which users can submit applications to individual queues. Also, there are safe-guards to ensure that users cannot view and/or modify applications from other users. Also, per-queue and system administrator roles are supported.
-* **Elasticity** - Free resources can be allocated to any queue beyond its capacity. When there is demand for these resources from queues running below capacity at a future point in time, as tasks scheduled on these resources complete, they will be assigned to applications on queues running below the capacity (pre-emption is also supported). This ensures that resources are available in a predictable and elastic manner to queues, thus preventing artifical silos of resources in the cluster which helps utilization.
+* **Elasticity** - Free resources can be allocated to any queue beyond its capacity. When there is demand for these resources from queues running below capacity at a future point in time, as tasks scheduled on these resources complete, they will be assigned to applications on queues running below the capacity (pre-emption is also supported). This ensures that resources are available in a predictable and elastic manner to queues, thus preventing artificial silos of resources in the cluster which helps utilization.
* **Multi-tenancy** - Comprehensive set of limits are provided to prevent a single application, user and queue from monopolizing resources of the queue or the cluster as a whole to ensure that the cluster isn't overwhelmed.
@@ -67,9 +67,9 @@ The `CapacityScheduler` supports the following features:
* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime.
- * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queueus*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues.
+ * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues.
-* **Resource-based Scheduling** - Support for resource-intensive applications, where-in a application can optionally specify higher resource-requirements than the default, there-by accomodating applications with differing resource requirements. Currently, *memory* is the resource requirement supported.
+* **Resource-based Scheduling** - Support for resource-intensive applications, where-in a application can optionally specify higher resource-requirements than the default, there-by accommodating applications with differing resource requirements. Currently, *memory* is the resource requirement supported.
* **Queue Mapping based on User or Group** - This feature allows users to map a job to a specific queue based on the user or group.
@@ -163,7 +163,7 @@ Configuration
| Property | Description |
|:---- |:---- |
-| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to aspecific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used.|
+| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used.|
| `yarn.scheduler.capacity.queue-mappings-override.enable` | This function is used to specify whether the user specified queues can be overridden. This is a Boolean value and the default value is *false*. |
Example:
[05/50] [abbrv] hadoop git commit: YARN-4822. Refactor existing
Preemption Policy of CS for easier adding new approach to select preemption
candidates. Contributed by Wangda Tan
Posted by ae...@apache.org.
YARN-4822. Refactor existing Preemption Policy of CS for easier adding new approach to select preemption candidates. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60e4116b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60e4116b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60e4116b
Branch: refs/heads/HDFS-1312
Commit: 60e4116bf1d00afed91010e57357fe54057e4e39
Parents: 09d63d5
Author: Jian He <ji...@apache.org>
Authored: Wed Mar 30 12:43:52 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Mar 30 12:43:52 2016 -0700
----------------------------------------------------------------------
.../monitor/SchedulingEditPolicy.java | 8 +-
.../monitor/SchedulingMonitor.java | 4 -
.../CapacitySchedulerPreemptionContext.java | 52 +
.../CapacitySchedulerPreemptionUtils.java | 65 ++
.../capacity/FifoCandidatesSelector.java | 364 ++++++
.../capacity/PreemptableResourceCalculator.java | 370 ++++++
.../capacity/PreemptionCandidatesSelector.java | 52 +
.../ProportionalCapacityPreemptionPolicy.java | 1086 ++++--------------
.../monitor/capacity/TempQueuePerPartition.java | 159 +++
.../CapacitySchedulerConfiguration.java | 45 +
.../capacity/preemption/PreemptableQueue.java | 6 -
.../capacity/preemption/PreemptionManager.java | 2 +-
...estProportionalCapacityPreemptionPolicy.java | 133 +--
...pacityPreemptionPolicyForNodePartitions.java | 78 +-
.../TestCapacitySchedulerPreemption.java | 14 +-
15 files changed, 1432 insertions(+), 1006 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
index 0d587d8..47458a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingEditPolicy.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResour
public interface SchedulingEditPolicy {
- public void init(Configuration config, RMContext context,
+ void init(Configuration config, RMContext context,
PreemptableResourceScheduler scheduler);
/**
@@ -31,10 +31,10 @@ public interface SchedulingEditPolicy {
* allowed to track containers and affect the scheduler. The "actions"
* performed are passed back through an EventHandler.
*/
- public void editSchedule();
+ void editSchedule();
- public long getMonitoringInterval();
+ long getMonitoringInterval();
- public String getPolicyName();
+ String getPolicyName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
index d4c129b..55ec858 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
@@ -45,10 +45,6 @@ public class SchedulingMonitor extends AbstractService {
this.rmContext = rmContext;
}
- public long getMonitorInterval() {
- return monitorInterval;
- }
-
@VisibleForTesting
public synchronized SchedulingEditPolicy getSchedulingEditPolicy() {
return scheduleEditPolicy;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
new file mode 100644
index 0000000..c52127d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
+import java.util.Collection;
+import java.util.Set;
+
+interface CapacitySchedulerPreemptionContext {
+ CapacityScheduler getScheduler();
+
+ TempQueuePerPartition getQueueByPartition(String queueName,
+ String partition);
+
+ Collection<TempQueuePerPartition> getQueuePartitions(String queueName);
+
+ ResourceCalculator getResourceCalculator();
+
+ RMContext getRMContext();
+
+ boolean isObserveOnly();
+
+ Set<ContainerId> getKillableContainers();
+
+ double getMaxIgnoreOverCapacity();
+
+ double getNaturalTerminationFactor();
+
+ Set<String> getLeafQueueNames();
+
+ Set<String> getAllPartitions();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
new file mode 100644
index 0000000..a71f108
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class CapacitySchedulerPreemptionUtils {
+ public static Map<String, Resource> getResToObtainByPartitionForLeafQueue(
+ CapacitySchedulerPreemptionContext context, String queueName,
+ Resource clusterResource) {
+ Map<String, Resource> resToObtainByPartition = new HashMap<>();
+ // compute resToObtainByPartition considered inter-queue preemption
+ for (TempQueuePerPartition qT : context.getQueuePartitions(queueName)) {
+ if (qT.preemptionDisabled) {
+ continue;
+ }
+
+ // Only add resToObtainByPartition when actuallyToBePreempted resource >= 0
+ if (Resources.greaterThan(context.getResourceCalculator(),
+ clusterResource, qT.actuallyToBePreempted, Resources.none())) {
+ resToObtainByPartition.put(qT.partition,
+ Resources.clone(qT.actuallyToBePreempted));
+ }
+ }
+
+ return resToObtainByPartition;
+ }
+
+ public static boolean isContainerAlreadySelected(RMContainer container,
+ Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates) {
+ if (null == selectedCandidates) {
+ return false;
+ }
+
+ Set<RMContainer> containers = selectedCandidates.get(
+ container.getApplicationAttemptId());
+ if (containers == null) {
+ return false;
+ }
+ return containers.contains(container);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
new file mode 100644
index 0000000..499d0ff
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class FifoCandidatesSelector
+ extends PreemptionCandidatesSelector {
+ private static final Log LOG =
+ LogFactory.getLog(FifoCandidatesSelector.class);
+ private PreemptableResourceCalculator preemptableAmountCalculator;
+
+ FifoCandidatesSelector(
+ CapacitySchedulerPreemptionContext preemptionContext) {
+ super(preemptionContext);
+
+ preemptableAmountCalculator = new PreemptableResourceCalculator(
+ preemptionContext);
+ }
+
+ @Override
+ public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
+ Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
+ Resource clusterResource, Resource totalPreemptionAllowed) {
+ // Calculate how much resources we need to preempt
+ preemptableAmountCalculator.computeIdealAllocation(clusterResource,
+ totalPreemptionAllowed);
+
+ Map<ApplicationAttemptId, Set<RMContainer>> preemptMap =
+ new HashMap<>();
+ List<RMContainer> skippedAMContainerlist = new ArrayList<>();
+
+ // Loop all leaf queues
+ for (String queueName : preemptionContext.getLeafQueueNames()) {
+ // check if preemption disabled for the queue
+ if (preemptionContext.getQueueByPartition(queueName,
+ RMNodeLabelsManager.NO_LABEL).preemptionDisabled) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("skipping from queue=" + queueName
+ + " because it's a non-preemptable queue");
+ }
+ continue;
+ }
+
+ // compute resToObtainByPartition considered inter-queue preemption
+ LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName,
+ RMNodeLabelsManager.NO_LABEL).leafQueue;
+
+ Map<String, Resource> resToObtainByPartition =
+ CapacitySchedulerPreemptionUtils
+ .getResToObtainByPartitionForLeafQueue(preemptionContext,
+ queueName, clusterResource);
+
+ synchronized (leafQueue) {
+ // go through all ignore-partition-exclusivity containers first to make
+ // sure such containers will be preemptionCandidates first
+ Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityContainers =
+ leafQueue.getIgnoreExclusivityRMContainers();
+ for (String partition : resToObtainByPartition.keySet()) {
+ if (ignorePartitionExclusivityContainers.containsKey(partition)) {
+ TreeSet<RMContainer> rmContainers =
+ ignorePartitionExclusivityContainers.get(partition);
+ // We will check container from reverse order, so latter submitted
+ // application's containers will be preemptionCandidates first.
+ for (RMContainer c : rmContainers.descendingSet()) {
+ if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c,
+ selectedCandidates)) {
+ // Skip already selected containers
+ continue;
+ }
+ boolean preempted = tryPreemptContainerAndDeductResToObtain(
+ resToObtainByPartition, c, clusterResource, preemptMap,
+ totalPreemptionAllowed);
+ if (!preempted) {
+ continue;
+ }
+ }
+ }
+ }
+
+ // preempt other containers
+ Resource skippedAMSize = Resource.newInstance(0, 0);
+ Iterator<FiCaSchedulerApp> desc =
+ leafQueue.getOrderingPolicy().getPreemptionIterator();
+ while (desc.hasNext()) {
+ FiCaSchedulerApp fc = desc.next();
+ // When we complete preempt from one partition, we will remove from
+ // resToObtainByPartition, so when it becomes empty, we can get no
+ // more preemption is needed
+ if (resToObtainByPartition.isEmpty()) {
+ break;
+ }
+
+ preemptFrom(fc, clusterResource, resToObtainByPartition,
+ skippedAMContainerlist, skippedAMSize, preemptMap,
+ totalPreemptionAllowed);
+ }
+
+ // Can try preempting AMContainers (still saving atmost
+ // maxAMCapacityForThisQueue AMResource's) if more resources are
+ // required to be preemptionCandidates from this Queue.
+ Resource maxAMCapacityForThisQueue = Resources.multiply(
+ Resources.multiply(clusterResource,
+ leafQueue.getAbsoluteCapacity()),
+ leafQueue.getMaxAMResourcePerQueuePercent());
+
+ preemptAMContainers(clusterResource, preemptMap, skippedAMContainerlist,
+ resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
+ totalPreemptionAllowed);
+ }
+ }
+
+ return preemptMap;
+ }
+
+ /**
+ * As more resources are needed for preemption, saved AMContainers has to be
+ * rescanned. Such AMContainers can be preemptionCandidates based on resToObtain, but
+ * maxAMCapacityForThisQueue resources will be still retained.
+ *
+ * @param clusterResource
+ * @param preemptMap
+ * @param skippedAMContainerlist
+ * @param skippedAMSize
+ * @param maxAMCapacityForThisQueue
+ */
+ private void preemptAMContainers(Resource clusterResource,
+ Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ List<RMContainer> skippedAMContainerlist,
+ Map<String, Resource> resToObtainByPartition, Resource skippedAMSize,
+ Resource maxAMCapacityForThisQueue, Resource totalPreemptionAllowed) {
+ for (RMContainer c : skippedAMContainerlist) {
+ // Got required amount of resources for preemption, can stop now
+ if (resToObtainByPartition.isEmpty()) {
+ break;
+ }
+ // Once skippedAMSize reaches down to maxAMCapacityForThisQueue,
+ // container selection iteration for preemption will be stopped.
+ if (Resources.lessThanOrEqual(rc, clusterResource, skippedAMSize,
+ maxAMCapacityForThisQueue)) {
+ break;
+ }
+
+ boolean preempted =
+ tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c,
+ clusterResource, preemptMap, totalPreemptionAllowed);
+ if (preempted) {
+ Resources.subtractFrom(skippedAMSize, c.getAllocatedResource());
+ }
+ }
+ skippedAMContainerlist.clear();
+ }
+
+ private boolean preemptMapContains(
+ Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ ApplicationAttemptId attemptId, RMContainer rmContainer) {
+ Set<RMContainer> rmContainers;
+ if (null == (rmContainers = preemptMap.get(attemptId))) {
+ return false;
+ }
+ return rmContainers.contains(rmContainer);
+ }
+
+ /**
+ * Return should we preempt rmContainer. If we should, deduct from
+ * <code>resourceToObtainByPartition</code>
+ */
+ private boolean tryPreemptContainerAndDeductResToObtain(
+ Map<String, Resource> resourceToObtainByPartitions,
+ RMContainer rmContainer, Resource clusterResource,
+ Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ Resource totalPreemptionAllowed) {
+ ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId();
+
+ // We will not account resource of a container twice or more
+ if (preemptMapContains(preemptMap, attemptId, rmContainer)) {
+ return false;
+ }
+
+ String nodePartition = getPartitionByNodeId(rmContainer.getAllocatedNode());
+ Resource toObtainByPartition =
+ resourceToObtainByPartitions.get(nodePartition);
+
+ if (null != toObtainByPartition && Resources.greaterThan(rc,
+ clusterResource, toObtainByPartition, Resources.none()) && Resources
+ .fitsIn(rc, clusterResource, rmContainer.getAllocatedResource(),
+ totalPreemptionAllowed)) {
+ Resources.subtractFrom(toObtainByPartition,
+ rmContainer.getAllocatedResource());
+ Resources.subtractFrom(totalPreemptionAllowed,
+ rmContainer.getAllocatedResource());
+
+ // When we have no more resource need to obtain, remove from map.
+ if (Resources.lessThanOrEqual(rc, clusterResource, toObtainByPartition,
+ Resources.none())) {
+ resourceToObtainByPartitions.remove(nodePartition);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Marked container=" + rmContainer.getContainerId()
+ + " in partition=" + nodePartition
+ + " to be preemption candidates");
+ }
+ // Add to preemptMap
+ addToPreemptMap(preemptMap, attemptId, rmContainer);
+ return true;
+ }
+
+ return false;
+ }
+
+ private String getPartitionByNodeId(NodeId nodeId) {
+ return preemptionContext.getScheduler().getSchedulerNode(nodeId)
+ .getPartition();
+ }
+
+ /**
+ * Given a target preemption for a specific application, select containers
+ * to preempt (after unreserving all reservation for that app).
+ */
+ @SuppressWarnings("unchecked")
+ private void preemptFrom(FiCaSchedulerApp app,
+ Resource clusterResource, Map<String, Resource> resToObtainByPartition,
+ List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
+ Map<ApplicationAttemptId, Set<RMContainer>> selectedContainers,
+ Resource totalPreemptionAllowed) {
+ ApplicationAttemptId appId = app.getApplicationAttemptId();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking at application=" + app.getApplicationAttemptId()
+ + " resourceToObtain=" + resToObtainByPartition);
+ }
+
+ // first drop reserved containers towards rsrcPreempt
+ List<RMContainer> reservedContainers =
+ new ArrayList<>(app.getReservedContainers());
+ for (RMContainer c : reservedContainers) {
+ if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c,
+ selectedContainers)) {
+ continue;
+ }
+ if (resToObtainByPartition.isEmpty()) {
+ return;
+ }
+
+ // Try to preempt this container
+ tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c,
+ clusterResource, selectedContainers, totalPreemptionAllowed);
+
+ if (!preemptionContext.isObserveOnly()) {
+ preemptionContext.getRMContext().getDispatcher().getEventHandler()
+ .handle(new ContainerPreemptEvent(appId, c,
+ SchedulerEventType.KILL_RESERVED_CONTAINER));
+ }
+ }
+
+ // if more resources are to be freed go through all live containers in
+ // reverse priority and reverse allocation order and mark them for
+ // preemption
+ List<RMContainer> liveContainers =
+ new ArrayList<>(app.getLiveContainers());
+
+ sortContainers(liveContainers);
+
+ for (RMContainer c : liveContainers) {
+ if (resToObtainByPartition.isEmpty()) {
+ return;
+ }
+
+ if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c,
+ selectedContainers)) {
+ continue;
+ }
+
+ // Skip already marked to killable containers
+ if (null != preemptionContext.getKillableContainers() && preemptionContext
+ .getKillableContainers().contains(c.getContainerId())) {
+ continue;
+ }
+
+ // Skip AM Container from preemption for now.
+ if (c.isAMContainer()) {
+ skippedAMContainerlist.add(c);
+ Resources.addTo(skippedAMSize, c.getAllocatedResource());
+ continue;
+ }
+
+ // Try to preempt this container
+ tryPreemptContainerAndDeductResToObtain(resToObtainByPartition, c,
+ clusterResource, selectedContainers, totalPreemptionAllowed);
+ }
+ }
+
+ /**
+ * Compare by reversed priority order first, and then reversed containerId
+ * order
+ * @param containers
+ */
+ @VisibleForTesting
+ static void sortContainers(List<RMContainer> containers){
+ Collections.sort(containers, new Comparator<RMContainer>() {
+ @Override
+ public int compare(RMContainer a, RMContainer b) {
+ Comparator<Priority> c = new org.apache.hadoop.yarn.server
+ .resourcemanager.resource.Priority.Comparator();
+ int priorityComp = c.compare(b.getContainer().getPriority(),
+ a.getContainer().getPriority());
+ if (priorityComp != 0) {
+ return priorityComp;
+ }
+ return b.getContainerId().compareTo(a.getContainerId());
+ }
+ });
+ }
+
+ private void addToPreemptMap(
+ Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ ApplicationAttemptId appAttemptId, RMContainer containerToPreempt) {
+ Set<RMContainer> set;
+ if (null == (set = preemptMap.get(appAttemptId))) {
+ set = new HashSet<>();
+ preemptMap.put(appAttemptId, set);
+ }
+ set.add(containerToPreempt);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
new file mode 100644
index 0000000..2217210
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
@@ -0,0 +1,370 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+/**
+ * Calculate how much resources need to be preempted for each queue,
+ * will be used by {@link FifoCandidatesSelector}
+ */
+public class PreemptableResourceCalculator {
+ private static final Log LOG =
+ LogFactory.getLog(PreemptableResourceCalculator.class);
+
+ private final CapacitySchedulerPreemptionContext context;
+ private final ResourceCalculator rc;
+
+ static class TQComparator implements Comparator<TempQueuePerPartition> {
+ private ResourceCalculator rc;
+ private Resource clusterRes;
+
+ TQComparator(ResourceCalculator rc, Resource clusterRes) {
+ this.rc = rc;
+ this.clusterRes = clusterRes;
+ }
+
+ @Override
+ public int compare(TempQueuePerPartition tq1, TempQueuePerPartition tq2) {
+ if (getIdealPctOfGuaranteed(tq1) < getIdealPctOfGuaranteed(tq2)) {
+ return -1;
+ }
+ if (getIdealPctOfGuaranteed(tq1) > getIdealPctOfGuaranteed(tq2)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ // Calculates idealAssigned / guaranteed
+ // TempQueues with 0 guarantees are always considered the most over
+ // capacity and therefore considered last for resources.
+ private double getIdealPctOfGuaranteed(TempQueuePerPartition q) {
+ double pctOver = Integer.MAX_VALUE;
+ if (q != null && Resources.greaterThan(
+ rc, clusterRes, q.guaranteed, Resources.none())) {
+ pctOver =
+ Resources.divide(rc, clusterRes, q.idealAssigned, q.guaranteed);
+ }
+ return (pctOver);
+ }
+ }
+
+ public PreemptableResourceCalculator(CapacitySchedulerPreemptionContext preemptionContext) {
+ context = preemptionContext;
+ rc = preemptionContext.getResourceCalculator();
+ }
+
+ /**
+ * Computes a normalizedGuaranteed capacity based on active queues
+ * @param rc resource calculator
+ * @param clusterResource the total amount of resources in the cluster
+ * @param queues the list of queues to consider
+ */
+ private void resetCapacity(ResourceCalculator rc, Resource clusterResource,
+ Collection<TempQueuePerPartition> queues, boolean ignoreGuar) {
+ Resource activeCap = Resource.newInstance(0, 0);
+
+ if (ignoreGuar) {
+ for (TempQueuePerPartition q : queues) {
+ q.normalizedGuarantee = 1.0f / queues.size();
+ }
+ } else {
+ for (TempQueuePerPartition q : queues) {
+ Resources.addTo(activeCap, q.guaranteed);
+ }
+ for (TempQueuePerPartition q : queues) {
+ q.normalizedGuarantee = Resources.divide(rc, clusterResource,
+ q.guaranteed, activeCap);
+ }
+ }
+ }
+
+ // Take the most underserved TempQueue (the one on the head). Collect and
+ // return the list of all queues that have the same idealAssigned
+ // percentage of guaranteed.
+ protected Collection<TempQueuePerPartition> getMostUnderservedQueues(
+ PriorityQueue<TempQueuePerPartition> orderedByNeed, TQComparator tqComparator) {
+ ArrayList<TempQueuePerPartition> underserved = new ArrayList<>();
+ while (!orderedByNeed.isEmpty()) {
+ TempQueuePerPartition q1 = orderedByNeed.remove();
+ underserved.add(q1);
+ TempQueuePerPartition q2 = orderedByNeed.peek();
+ // q1's pct of guaranteed won't be larger than q2's. If it's less, then
+ // return what has already been collected. Otherwise, q1's pct of
+ // guaranteed == that of q2, so add q2 to underserved list during the
+ // next pass.
+ if (q2 == null || tqComparator.compare(q1,q2) < 0) {
+ return underserved;
+ }
+ }
+ return underserved;
+ }
+
+
+ /**
+ * Given a set of queues compute the fix-point distribution of unassigned
+ * resources among them. As pending request of a queue are exhausted, the
+ * queue is removed from the set and remaining capacity redistributed among
+ * remaining queues. The distribution is weighted based on guaranteed
+ * capacity, unless asked to ignoreGuarantee, in which case resources are
+ * distributed uniformly.
+ */
+ private void computeFixpointAllocation(ResourceCalculator rc,
+ Resource tot_guarant, Collection<TempQueuePerPartition> qAlloc,
+ Resource unassigned, boolean ignoreGuarantee) {
+ // Prior to assigning the unused resources, process each queue as follows:
+ // If current > guaranteed, idealAssigned = guaranteed + untouchable extra
+ // Else idealAssigned = current;
+ // Subtract idealAssigned resources from unassigned.
+ // If the queue has all of its needs met (that is, if
+ // idealAssigned >= current + pending), remove it from consideration.
+ // Sort queues from most under-guaranteed to most over-guaranteed.
+ TQComparator tqComparator = new TQComparator(rc, tot_guarant);
+ PriorityQueue<TempQueuePerPartition> orderedByNeed = new PriorityQueue<>(10,
+ tqComparator);
+ for (Iterator<TempQueuePerPartition> i = qAlloc.iterator(); i.hasNext();) {
+ TempQueuePerPartition q = i.next();
+ if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) {
+ q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra);
+ } else {
+ q.idealAssigned = Resources.clone(q.current);
+ }
+ Resources.subtractFrom(unassigned, q.idealAssigned);
+ // If idealAssigned < (current + pending), q needs more resources, so
+ // add it to the list of underserved queues, ordered by need.
+ Resource curPlusPend = Resources.add(q.current, q.pending);
+ if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) {
+ orderedByNeed.add(q);
+ }
+ }
+
+ //assign all cluster resources until no more demand, or no resources are left
+ while (!orderedByNeed.isEmpty()
+ && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) {
+ Resource wQassigned = Resource.newInstance(0, 0);
+ // we compute normalizedGuarantees capacity based on currently active
+ // queues
+ resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee);
+
+ // For each underserved queue (or set of queues if multiple are equally
+ // underserved), offer its share of the unassigned resources based on its
+ // normalized guarantee. After the offer, if the queue is not satisfied,
+ // place it back in the ordered list of queues, recalculating its place
+ // in the order of most under-guaranteed to most over-guaranteed. In this
+ // way, the most underserved queue(s) are always given resources first.
+ Collection<TempQueuePerPartition> underserved =
+ getMostUnderservedQueues(orderedByNeed, tqComparator);
+ for (Iterator<TempQueuePerPartition> i = underserved.iterator(); i
+ .hasNext();) {
+ TempQueuePerPartition sub = i.next();
+ Resource wQavail = Resources.multiplyAndNormalizeUp(rc,
+ unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1));
+ Resource wQidle = sub.offer(wQavail, rc, tot_guarant);
+ Resource wQdone = Resources.subtract(wQavail, wQidle);
+
+ if (Resources.greaterThan(rc, tot_guarant,
+ wQdone, Resources.none())) {
+ // The queue is still asking for more. Put it back in the priority
+ // queue, recalculating its order based on need.
+ orderedByNeed.add(sub);
+ }
+ Resources.addTo(wQassigned, wQdone);
+ }
+ Resources.subtractFrom(unassigned, wQassigned);
+ }
+ }
+
+ /**
+ * This method computes (for a single level in the tree, passed as a {@code
+ * List<TempQueue>}) the ideal assignment of resources. This is done
+ * recursively to allocate capacity fairly across all queues with pending
+ * demands. It terminates when no resources are left to assign, or when all
+ * demand is satisfied.
+ *
+ * @param rc resource calculator
+ * @param queues a list of cloned queues to be assigned capacity to (this is
+ * an out param)
+ * @param totalPreemptionAllowed total amount of preemption we allow
+ * @param tot_guarant the amount of capacity assigned to this pool of queues
+ */
+ private void computeIdealResourceDistribution(ResourceCalculator rc,
+ List<TempQueuePerPartition> queues, Resource totalPreemptionAllowed,
+ Resource tot_guarant) {
+
+ // qAlloc tracks currently active queues (will decrease progressively as
+ // demand is met)
+ List<TempQueuePerPartition> qAlloc = new ArrayList<>(queues);
+ // unassigned tracks how much resources are still to assign, initialized
+ // with the total capacity for this set of queues
+ Resource unassigned = Resources.clone(tot_guarant);
+
+ // group queues based on whether they have non-zero guaranteed capacity
+ Set<TempQueuePerPartition> nonZeroGuarQueues = new HashSet<>();
+ Set<TempQueuePerPartition> zeroGuarQueues = new HashSet<>();
+
+ for (TempQueuePerPartition q : qAlloc) {
+ if (Resources
+ .greaterThan(rc, tot_guarant, q.guaranteed, Resources.none())) {
+ nonZeroGuarQueues.add(q);
+ } else {
+ zeroGuarQueues.add(q);
+ }
+ }
+
+ // first compute the allocation as a fixpoint based on guaranteed capacity
+ computeFixpointAllocation(rc, tot_guarant, nonZeroGuarQueues, unassigned,
+ false);
+
+ // if any capacity is left unassigned, distributed among zero-guarantee
+ // queues uniformly (i.e., not based on guaranteed capacity, as this is zero)
+ if (!zeroGuarQueues.isEmpty()
+ && Resources.greaterThan(rc, tot_guarant, unassigned, Resources.none())) {
+ computeFixpointAllocation(rc, tot_guarant, zeroGuarQueues, unassigned,
+ true);
+ }
+
+ // based on ideal assignment computed above and current assignment we derive
+ // how much preemption is required overall
+ Resource totPreemptionNeeded = Resource.newInstance(0, 0);
+ for (TempQueuePerPartition t:queues) {
+ if (Resources.greaterThan(rc, tot_guarant, t.current, t.idealAssigned)) {
+ Resources.addTo(totPreemptionNeeded,
+ Resources.subtract(t.current, t.idealAssigned));
+ }
+ }
+
+ // if we need to preempt more than is allowed, compute a factor (0<f<1)
+ // that is used to scale down how much we ask back from each queue
+ float scalingFactor = 1.0F;
+ if (Resources.greaterThan(rc, tot_guarant,
+ totPreemptionNeeded, totalPreemptionAllowed)) {
+ scalingFactor = Resources.divide(rc, tot_guarant,
+ totalPreemptionAllowed, totPreemptionNeeded);
+ }
+
+ // assign to each queue the amount of actual preemption based on local
+ // information of ideal preemption and scaling factor
+ for (TempQueuePerPartition t : queues) {
+ t.assignPreemption(scalingFactor, rc, tot_guarant);
+ }
+ if (LOG.isDebugEnabled()) {
+ for (TempQueuePerPartition t : queues) {
+ LOG.debug(t);
+ }
+ }
+
+ }
+
+ /**
+ * This method recursively computes the ideal assignment of resources to each
+ * level of the hierarchy. This ensures that leafs that are over-capacity but
+ * with parents within capacity will not be preemptionCandidates. Preemptions are allowed
+ * within each subtree according to local over/under capacity.
+ *
+ * @param root the root of the cloned queue hierachy
+ * @param totalPreemptionAllowed maximum amount of preemption allowed
+ * @return a list of leaf queues updated with preemption targets
+ */
+ private void recursivelyComputeIdealAssignment(
+ TempQueuePerPartition root, Resource totalPreemptionAllowed) {
+ if (root.getChildren() != null &&
+ root.getChildren().size() > 0) {
+ // compute ideal distribution at this level
+ computeIdealResourceDistribution(rc, root.getChildren(),
+ totalPreemptionAllowed, root.idealAssigned);
+ // compute recursively for lower levels and build list of leafs
+ for(TempQueuePerPartition t : root.getChildren()) {
+ recursivelyComputeIdealAssignment(t, totalPreemptionAllowed);
+ }
+ }
+ }
+
+
+ private void calculateResToObtainByPartitionForLeafQueues(
+ Set<String> leafQueueNames, Resource clusterResource) {
+ // Loop all leaf queues
+ for (String queueName : leafQueueNames) {
+ // check if preemption disabled for the queue
+ if (context.getQueueByPartition(queueName,
+ RMNodeLabelsManager.NO_LABEL).preemptionDisabled) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("skipping from queue=" + queueName
+ + " because it's a non-preemptable queue");
+ }
+ continue;
+ }
+
+ // compute resToObtainByPartition considered inter-queue preemption
+ for (TempQueuePerPartition qT : context.getQueuePartitions(queueName)) {
+ // we act only if we are violating balance by more than
+ // maxIgnoredOverCapacity
+ if (Resources.greaterThan(rc, clusterResource, qT.current,
+ Resources.multiply(qT.guaranteed, 1.0 + context.getMaxIgnoreOverCapacity()))) {
+ // we introduce a dampening factor naturalTerminationFactor that
+ // accounts for natural termination of containers
+ Resource resToObtain = Resources.multiply(qT.toBePreempted,
+ context.getNaturalTerminationFactor());
+ // Only add resToObtain when it >= 0
+ if (Resources.greaterThan(rc, clusterResource, resToObtain,
+ Resources.none())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Queue=" + queueName + " partition=" + qT.partition
+ + " resource-to-obtain=" + resToObtain);
+ }
+ }
+ qT.actuallyToBePreempted = Resources.clone(resToObtain);
+ } else {
+ qT.actuallyToBePreempted = Resources.none();
+ }
+ }
+ }
+ }
+
+ public void computeIdealAllocation(Resource clusterResource,
+ Resource totalPreemptionAllowed) {
+ for (String partition : context.getAllPartitions()) {
+ TempQueuePerPartition tRoot =
+ context.getQueueByPartition(CapacitySchedulerConfiguration.ROOT, partition);
+ // compute the ideal distribution of resources among queues
+ // updates cloned queues state accordingly
+ tRoot.idealAssigned = tRoot.guaranteed;
+ recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed);
+ }
+
+ // based on ideal allocation select containers to be preempted from each
+ // calculate resource-to-obtain by partition for each leaf queues
+ calculateResToObtainByPartitionForLeafQueues(context.getLeafQueueNames(),
+ clusterResource);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60e4116b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
new file mode 100644
index 0000000..dd33d8f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
+import java.util.Map;
+import java.util.Set;
+
+public abstract class PreemptionCandidatesSelector {
+ protected CapacitySchedulerPreemptionContext preemptionContext;
+ protected ResourceCalculator rc;
+
+ PreemptionCandidatesSelector(
+ CapacitySchedulerPreemptionContext preemptionContext) {
+ this.preemptionContext = preemptionContext;
+ this.rc = preemptionContext.getResourceCalculator();
+ }
+
+ /**
+ * Get preemption candidates from computed resource sharing and already
+ * selected candidates.
+ *
+ * @param selectedCandidates already selected candidates from previous policies
+ * @param clusterResource
+ * @param totalPreemptedResourceAllowed how many resources allowed to be
+ * preempted in this round
+ * @return merged selected candidates.
+ */
+ public abstract Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
+ Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
+ Resource clusterResource, Resource totalPreemptedResourceAllowed);
+}
[28/50] [abbrv] hadoop git commit: HDFS-10253. Fix
TestRefreshCallQueue failure (Contributed by Xiaoyu Yao)
Posted by ae...@apache.org.
HDFS-10253. Fix TestRefreshCallQueue failure (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54b2e78f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54b2e78f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54b2e78f
Branch: refs/heads/HDFS-1312
Commit: 54b2e78fd28c9def42bec7f0418833bad352686c
Parents: 81d04ca
Author: Vinayakumar B <vi...@apache.org>
Authored: Sun Apr 3 13:27:49 2016 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Sun Apr 3 13:27:49 2016 +0530
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/TestRefreshCallQueue.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54b2e78f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
index 1be2752..5cb7def 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
@@ -92,7 +92,7 @@ public class TestRefreshCallQueue {
@SuppressWarnings("serial")
public static class MockCallQueue<E> extends LinkedBlockingQueue<E> {
- public MockCallQueue(int cap, String ns, Configuration conf) {
+ public MockCallQueue(int levels, int cap, String ns, Configuration conf) {
super(cap);
mockQueueConstructions++;
}
[46/50] [abbrv] hadoop git commit: YARN-4916.
TestNMProxy.tesNMProxyRPCRetry fails. Contributed by Tibor Kiss.
Posted by ae...@apache.org.
YARN-4916. TestNMProxy.tesNMProxyRPCRetry fails. Contributed by Tibor Kiss.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00058167
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00058167
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00058167
Branch: refs/heads/HDFS-1312
Commit: 00058167431475c6e63c80207424f1d365569e3a
Parents: 9174645
Author: Junping Du <ju...@apache.org>
Authored: Tue Apr 5 09:01:08 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Tue Apr 5 09:01:08 2016 -0700
----------------------------------------------------------------------
.../yarn/server/nodemanager/containermanager/TestNMProxy.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00058167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index 7ce15c5..46b32de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -188,8 +188,7 @@ public class TestNMProxy extends BaseContainerManagerTest {
Assert.fail("should get socket exception");
} catch (IOException e) {
// socket exception should be thrown immediately, without RPC retries.
- Assert.assertTrue(e.toString().
- contains("Failed on local exception: java.net.SocketException"));
+ Assert.assertTrue(e instanceof java.net.SocketException);
}
}
[15/50] [abbrv] hadoop git commit: HADOOP-12916. Allow RPC
scheduler/callqueue backoff using response times. Contributed by Xiaoyu Yao.
Posted by ae...@apache.org.
HADOOP-12916. Allow RPC scheduler/callqueue backoff using response times. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d95c6eb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d95c6eb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d95c6eb3
Branch: refs/heads/HDFS-1312
Commit: d95c6eb32cec7768ac418fb467b1198ccf3cf0dc
Parents: 0a74610
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Mar 31 08:42:57 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Mar 31 08:42:57 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 13 +
.../hadoop/fs/CommonConfigurationKeys.java | 14 +-
.../org/apache/hadoop/ipc/CallQueueManager.java | 124 +++++-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 396 +++++++++++++++----
.../hadoop/ipc/DecayRpcSchedulerMXBean.java | 2 +
.../apache/hadoop/ipc/DefaultRpcScheduler.java | 45 +++
.../org/apache/hadoop/ipc/FairCallQueue.java | 45 +--
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 8 +-
.../org/apache/hadoop/ipc/RpcScheduler.java | 8 +-
.../java/org/apache/hadoop/ipc/Schedulable.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 77 +++-
.../apache/hadoop/ipc/WritableRpcEngine.java | 45 +--
.../apache/hadoop/ipc/TestCallQueueManager.java | 147 ++++++-
.../hadoop/ipc/TestDecayRpcScheduler.java | 42 +-
.../apache/hadoop/ipc/TestFairCallQueue.java | 79 ++--
.../hadoop/ipc/TestIdentityProviders.java | 18 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 92 ++++-
17 files changed, 893 insertions(+), 267 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 8355d96..4c8f27b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -1626,6 +1626,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return defaultValue;
}
vStr = vStr.trim();
+ return getTimeDurationHelper(name, vStr, unit);
+ }
+
+ private long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
@@ -1636,6 +1640,15 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return unit.convert(Long.parseLong(vStr), vUnit.unit());
}
+ public long[] getTimeDurations(String name, TimeUnit unit) {
+ String[] strings = getTrimmedStrings(name);
+ long[] durations = new long[strings.length];
+ for (int i = 0; i < strings.length; i++) {
+ durations[i] = getTimeDurationHelper(name, strings[i], unit);
+ }
+ return durations;
+ }
+
/**
* Get the value of the <code>name</code> property as a <code>Pattern</code>.
* If no such property is specified, or if the specified value is not a valid
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 9b4069a..a708900 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -90,14 +90,22 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
/**
* CallQueue related settings. These are not used directly, but rather
* combined with a namespace and port. For instance:
- * IPC_CALLQUEUE_NAMESPACE + ".8020." + IPC_CALLQUEUE_IMPL_KEY
+ * IPC_NAMESPACE + ".8020." + IPC_CALLQUEUE_IMPL_KEY
*/
- public static final String IPC_CALLQUEUE_NAMESPACE = "ipc";
+ public static final String IPC_NAMESPACE = "ipc";
public static final String IPC_CALLQUEUE_IMPL_KEY = "callqueue.impl";
- public static final String IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY = "identity-provider.impl";
+ public static final String IPC_SCHEDULER_IMPL_KEY = "scheduler.impl";
+ public static final String IPC_IDENTITY_PROVIDER_KEY = "identity-provider.impl";
public static final String IPC_BACKOFF_ENABLE = "backoff.enable";
public static final boolean IPC_BACKOFF_ENABLE_DEFAULT = false;
+ /**
+ * IPC scheduler priority levels.
+ */
+ public static final String IPC_SCHEDULER_PRIORITY_LEVELS_KEY =
+ "scheduler.priority.levels";
+ public static final int IPC_SCHEDULER_PRIORITY_LEVELS_DEFAULT_KEY = 4;
+
/** This is for specifying the implementation for the mappings from
* hostnames to the racks they belong to
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index 2ee15d3..1a7782a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
/**
* Abstracts queue operations for different blocking queues.
@@ -43,6 +44,13 @@ public class CallQueueManager<E> {
Class<?> queueClass, Class<E> elementClass) {
return (Class<? extends BlockingQueue<E>>)queueClass;
}
+
+ @SuppressWarnings("unchecked")
+ static Class<? extends RpcScheduler> convertSchedulerClass(
+ Class<?> schedulerClass) {
+ return (Class<? extends RpcScheduler>)schedulerClass;
+ }
+
private final boolean clientBackOffEnabled;
// Atomic refs point to active callQueue
@@ -50,25 +58,76 @@ public class CallQueueManager<E> {
private final AtomicReference<BlockingQueue<E>> putRef;
private final AtomicReference<BlockingQueue<E>> takeRef;
+ private RpcScheduler scheduler;
+
public CallQueueManager(Class<? extends BlockingQueue<E>> backingClass,
+ Class<? extends RpcScheduler> schedulerClass,
boolean clientBackOffEnabled, int maxQueueSize, String namespace,
Configuration conf) {
+ int priorityLevels = parseNumLevels(namespace, conf);
+ this.scheduler = createScheduler(schedulerClass, priorityLevels,
+ namespace, conf);
BlockingQueue<E> bq = createCallQueueInstance(backingClass,
- maxQueueSize, namespace, conf);
+ priorityLevels, maxQueueSize, namespace, conf);
this.clientBackOffEnabled = clientBackOffEnabled;
this.putRef = new AtomicReference<BlockingQueue<E>>(bq);
this.takeRef = new AtomicReference<BlockingQueue<E>>(bq);
- LOG.info("Using callQueue " + backingClass);
+ LOG.info("Using callQueue: " + backingClass + " scheduler: " +
+ schedulerClass);
+ }
+
+ private static <T extends RpcScheduler> T createScheduler(
+ Class<T> theClass, int priorityLevels, String ns, Configuration conf) {
+ // Used for custom, configurable scheduler
+ try {
+ Constructor<T> ctor = theClass.getDeclaredConstructor(int.class,
+ String.class, Configuration.class);
+ return ctor.newInstance(priorityLevels, ns, conf);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (InvocationTargetException e) {
+ throw new RuntimeException(theClass.getName()
+ + " could not be constructed.", e.getCause());
+ } catch (Exception e) {
+ }
+
+ try {
+ Constructor<T> ctor = theClass.getDeclaredConstructor(int.class);
+ return ctor.newInstance(priorityLevels);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (InvocationTargetException e) {
+ throw new RuntimeException(theClass.getName()
+ + " could not be constructed.", e.getCause());
+ } catch (Exception e) {
+ }
+
+ // Last attempt
+ try {
+ Constructor<T> ctor = theClass.getDeclaredConstructor();
+ return ctor.newInstance();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (InvocationTargetException e) {
+ throw new RuntimeException(theClass.getName()
+ + " could not be constructed.", e.getCause());
+ } catch (Exception e) {
+ }
+
+ // Nothing worked
+ throw new RuntimeException(theClass.getName() +
+ " could not be constructed.");
}
private <T extends BlockingQueue<E>> T createCallQueueInstance(
- Class<T> theClass, int maxLen, String ns, Configuration conf) {
+ Class<T> theClass, int priorityLevels, int maxLen, String ns,
+ Configuration conf) {
// Used for custom, configurable callqueues
try {
- Constructor<T> ctor = theClass.getDeclaredConstructor(int.class, String.class,
- Configuration.class);
- return ctor.newInstance(maxLen, ns, conf);
+ Constructor<T> ctor = theClass.getDeclaredConstructor(int.class,
+ int.class, String.class, Configuration.class);
+ return ctor.newInstance(priorityLevels, maxLen, ns, conf);
} catch (RuntimeException e) {
throw e;
} catch (InvocationTargetException e) {
@@ -110,6 +169,22 @@ public class CallQueueManager<E> {
return clientBackOffEnabled;
}
+ // Based on policy to determine back off current call
+ boolean shouldBackOff(Schedulable e) {
+ return scheduler.shouldBackOff(e);
+ }
+
+ void addResponseTime(String name, int priorityLevel, int queueTime,
+ int processingTime) {
+ scheduler.addResponseTime(name, priorityLevel, queueTime, processingTime);
+ }
+
+ // This should be only called once per call and cached in the call object
+ // each getPriorityLevel call will increment the counter for the caller
+ int getPriorityLevel(Schedulable e) {
+ return scheduler.getPriorityLevel(e);
+ }
+
/**
* Insert e into the backing queue or block until we can.
* If we block and the queue changes on us, we will insert while the
@@ -147,14 +222,45 @@ public class CallQueueManager<E> {
}
/**
+ * Read the number of levels from the configuration.
+ * This will affect the FairCallQueue's overall capacity.
+ * @throws IllegalArgumentException on invalid queue count
+ */
+ @SuppressWarnings("deprecation")
+ private static int parseNumLevels(String ns, Configuration conf) {
+ // Fair call queue levels (IPC_CALLQUEUE_PRIORITY_LEVELS_KEY)
+ // takes priority over the scheduler level key
+ // (IPC_SCHEDULER_PRIORITY_LEVELS_KEY)
+ int retval = conf.getInt(ns + "." +
+ FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 0);
+ if (retval == 0) { // No FCQ priority level configured
+ retval = conf.getInt(ns + "." +
+ CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_KEY,
+ CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_DEFAULT_KEY);
+ } else {
+ LOG.warn(ns + "." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY +
+ " is deprecated. Please use " + ns + "." +
+ CommonConfigurationKeys.IPC_SCHEDULER_PRIORITY_LEVELS_KEY + ".");
+ }
+ if(retval < 1) {
+ throw new IllegalArgumentException("numLevels must be at least 1");
+ }
+ return retval;
+ }
+
+ /**
* Replaces active queue with the newly requested one and transfers
* all calls to the newQ before returning.
*/
public synchronized void swapQueue(
+ Class<? extends RpcScheduler> schedulerClass,
Class<? extends BlockingQueue<E>> queueClassToUse, int maxSize,
String ns, Configuration conf) {
- BlockingQueue<E> newQ = createCallQueueInstance(queueClassToUse, maxSize,
- ns, conf);
+ int priorityLevels = parseNumLevels(ns, conf);
+ RpcScheduler newScheduler = createScheduler(schedulerClass, priorityLevels,
+ ns, conf);
+ BlockingQueue<E> newQ = createCallQueueInstance(queueClassToUse,
+ priorityLevels, maxSize, ns, conf);
// Our current queue becomes the old queue
BlockingQueue<E> oldQ = putRef.get();
@@ -168,6 +274,8 @@ public class CallQueueManager<E> {
// Swap takeRef to handle new calls
takeRef.set(newQ);
+ this.scheduler = newScheduler;
+
LOG.info("Old Queue: " + stringRepr(oldQ) + ", " +
"Replacement: " + stringRepr(newQ));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index a6a14d0..4237339 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -27,17 +27,21 @@ import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import com.google.common.util.concurrent.AtomicDoubleArray;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.metrics2.util.MBeans;
import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The decay RPC scheduler counts incoming requests in a map, then
@@ -49,22 +53,28 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
/**
* Period controls how many milliseconds between each decay sweep.
*/
- public static final String IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY =
+ public static final String IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY =
+ "decay-scheduler.period-ms";
+ public static final long IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT =
+ 5000;
+ @Deprecated
+ public static final String IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY =
"faircallqueue.decay-scheduler.period-ms";
- public static final long IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT =
- 5000L;
/**
* Decay factor controls how much each count is suppressed by on each sweep.
* Valid numbers are > 0 and < 1. Decay factor works in tandem with period
* to control how long the scheduler remembers an identity.
*/
- public static final String IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY =
+ public static final String IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY =
+ "decay-scheduler.decay-factor";
+ public static final double IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT =
+ 0.5;
+ @Deprecated
+ public static final String IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY =
"faircallqueue.decay-scheduler.decay-factor";
- public static final double IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT =
- 0.5;
- /**
+ /**
* Thresholds are specified as integer percentages, and specify which usage
* range each queue will be allocated to. For instance, specifying the list
* 10, 40, 80
@@ -74,15 +84,31 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
* - q1 from 10 up to 40
* - q0 otherwise.
*/
- public static final String IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY =
- "faircallqueue.decay-scheduler.thresholds";
+ public static final String IPC_DECAYSCHEDULER_THRESHOLDS_KEY =
+ "decay-scheduler.thresholds";
+ @Deprecated
+ public static final String IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY =
+ "faircallqueue.decay-scheduler.thresholds";
// Specifies the identity to use when the IdentityProvider cannot handle
// a schedulable.
public static final String DECAYSCHEDULER_UNKNOWN_IDENTITY =
- "IdentityProvider.Unknown";
+ "IdentityProvider.Unknown";
+
+ public static final String
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY =
+ "decay-scheduler.backoff.responsetime.enable";
+ public static final Boolean
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_DEFAULT = false;
+
+ // Specifies the average response time (ms) thresholds of each
+ // level to trigger backoff
+ public static final String
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY =
+ "decay-scheduler.backoff.responsetime.thresholds";
- public static final Log LOG = LogFactory.getLog(DecayRpcScheduler.class);
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DecayRpcScheduler.class);
// Track the number of calls for each schedulable identity
private final ConcurrentHashMap<Object, AtomicLong> callCounts =
@@ -91,6 +117,14 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
// Should be the sum of all AtomicLongs in callCounts
private final AtomicLong totalCalls = new AtomicLong();
+ // Track total call count and response time in current decay window
+ private final AtomicLongArray responseTimeCountInCurrWindow;
+ private final AtomicLongArray responseTimeTotalInCurrWindow;
+
+ // Track average response time in previous decay window
+ private final AtomicDoubleArray responseTimeAvgInLastWindow;
+ private final AtomicLongArray responseTimeCountInLastWindow;
+
// Pre-computed scheduling decisions during the decay sweep are
// atomically swapped in as a read-only map
private final AtomicReference<Map<Object, Integer>> scheduleCacheRef =
@@ -98,10 +132,12 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
// Tune the behavior of the scheduler
private final long decayPeriodMillis; // How long between each tick
- private final double decayFactor; // nextCount = currentCount / decayFactor
- private final int numQueues; // affects scheduling decisions, from 0 to numQueues - 1
+ private final double decayFactor; // nextCount = currentCount * decayFactor
+ private final int numLevels;
private final double[] thresholds;
private final IdentityProvider identityProvider;
+ private final boolean backOffByResponseTimeEnabled;
+ private final long[] backOffResponseTimeThresholds;
/**
* This TimerTask will call decayCurrentCounts until
@@ -132,35 +168,46 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
/**
* Create a decay scheduler.
- * @param numQueues number of queues to schedule for
+ * @param numLevels number of priority levels
* @param ns config prefix, so that we can configure multiple schedulers
* in a single instance.
* @param conf configuration to use.
*/
- public DecayRpcScheduler(int numQueues, String ns, Configuration conf) {
- if (numQueues < 1) {
- throw new IllegalArgumentException("number of queues must be > 0");
+ public DecayRpcScheduler(int numLevels, String ns, Configuration conf) {
+ if(numLevels < 1) {
+ throw new IllegalArgumentException("Number of Priority Levels must be " +
+ "at least 1");
}
-
- this.numQueues = numQueues;
+ this.numLevels = numLevels;
this.decayFactor = parseDecayFactor(ns, conf);
this.decayPeriodMillis = parseDecayPeriodMillis(ns, conf);
this.identityProvider = this.parseIdentityProvider(ns, conf);
- this.thresholds = parseThresholds(ns, conf, numQueues);
+ this.thresholds = parseThresholds(ns, conf, numLevels);
+ this.backOffByResponseTimeEnabled = parseBackOffByResponseTimeEnabled(ns,
+ conf);
+ this.backOffResponseTimeThresholds =
+ parseBackOffResponseTimeThreshold(ns, conf, numLevels);
// Setup delay timer
Timer timer = new Timer();
DecayTask task = new DecayTask(this, timer);
timer.scheduleAtFixedRate(task, decayPeriodMillis, decayPeriodMillis);
- MetricsProxy prox = MetricsProxy.getInstance(ns);
+ // Setup response time metrics
+ responseTimeTotalInCurrWindow = new AtomicLongArray(numLevels);
+ responseTimeCountInCurrWindow = new AtomicLongArray(numLevels);
+ responseTimeAvgInLastWindow = new AtomicDoubleArray(numLevels);
+ responseTimeCountInLastWindow = new AtomicLongArray(numLevels);
+
+ MetricsProxy prox = MetricsProxy.getInstance(ns, numLevels);
prox.setDelegate(this);
}
// Load configs
- private IdentityProvider parseIdentityProvider(String ns, Configuration conf) {
+ private IdentityProvider parseIdentityProvider(String ns,
+ Configuration conf) {
List<IdentityProvider> providers = conf.getInstances(
- ns + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY,
+ ns + "." + CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY,
IdentityProvider.class);
if (providers.size() < 1) {
@@ -174,10 +221,16 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
private static double parseDecayFactor(String ns, Configuration conf) {
double factor = conf.getDouble(ns + "." +
- IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
- IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
- );
-
+ IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, 0.0);
+ if (factor == 0.0) {
+ factor = conf.getDouble(ns + "." +
+ IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY,
+ IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT);
+ } else if ((factor > 0.0) && (factor < 1)) {
+ LOG.warn(IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY +
+ " is deprecated. Please use " +
+ IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY + ".");
+ }
if (factor <= 0 || factor >= 1) {
throw new IllegalArgumentException("Decay Factor " +
"must be between 0 and 1");
@@ -188,10 +241,17 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
private static long parseDecayPeriodMillis(String ns, Configuration conf) {
long period = conf.getLong(ns + "." +
- IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY,
- IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT
- );
-
+ IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY,
+ 0);
+ if (period == 0) {
+ period = conf.getLong(ns + "." +
+ IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY,
+ IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT);
+ } else if (period > 0) {
+ LOG.warn((IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY +
+ " is deprecated. Please use " +
+ IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_KEY));
+ }
if (period <= 0) {
throw new IllegalArgumentException("Period millis must be >= 0");
}
@@ -200,15 +260,24 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
}
private static double[] parseThresholds(String ns, Configuration conf,
- int numQueues) {
+ int numLevels) {
int[] percentages = conf.getInts(ns + "." +
- IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY);
+ IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY);
if (percentages.length == 0) {
- return getDefaultThresholds(numQueues);
- } else if (percentages.length != numQueues-1) {
+ percentages = conf.getInts(ns + "." + IPC_DECAYSCHEDULER_THRESHOLDS_KEY);
+ if (percentages.length == 0) {
+ return getDefaultThresholds(numLevels);
+ }
+ } else {
+ LOG.warn(IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY +
+ " is deprecated. Please use " +
+ IPC_DECAYSCHEDULER_THRESHOLDS_KEY);
+ }
+
+ if (percentages.length != numLevels-1) {
throw new IllegalArgumentException("Number of thresholds should be " +
- (numQueues-1) + ". Was: " + percentages.length);
+ (numLevels-1) + ". Was: " + percentages.length);
}
// Convert integer percentages to decimals
@@ -223,14 +292,14 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
/**
* Generate default thresholds if user did not specify. Strategy is
* to halve each time, since queue usage tends to be exponential.
- * So if numQueues is 4, we would generate: double[]{0.125, 0.25, 0.5}
+ * So if numLevels is 4, we would generate: double[]{0.125, 0.25, 0.5}
* which specifies the boundaries between each queue's usage.
- * @param numQueues number of queues to compute for
- * @return array of boundaries of length numQueues - 1
+ * @param numLevels number of levels to compute for
+ * @return array of boundaries of length numLevels - 1
*/
- private static double[] getDefaultThresholds(int numQueues) {
- double[] ret = new double[numQueues - 1];
- double div = Math.pow(2, numQueues - 1);
+ private static double[] getDefaultThresholds(int numLevels) {
+ double[] ret = new double[numLevels - 1];
+ double div = Math.pow(2, numLevels - 1);
for (int i = 0; i < ret.length; i++) {
ret[i] = Math.pow(2, i)/div;
@@ -238,39 +307,89 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
return ret;
}
+ private static long[] parseBackOffResponseTimeThreshold(String ns,
+ Configuration conf, int numLevels) {
+ long[] responseTimeThresholds = conf.getTimeDurations(ns + "." +
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_THRESHOLDS_KEY,
+ TimeUnit.MILLISECONDS);
+ // backoff thresholds not specified
+ if (responseTimeThresholds.length == 0) {
+ return getDefaultBackOffResponseTimeThresholds(numLevels);
+ }
+ // backoff thresholds specified but not match with the levels
+ if (responseTimeThresholds.length != numLevels) {
+ throw new IllegalArgumentException(
+ "responseTimeThresholds must match with the number of priority " +
+ "levels");
+ }
+ // invalid thresholds
+ for (long responseTimeThreshold: responseTimeThresholds) {
+ if (responseTimeThreshold <= 0) {
+ throw new IllegalArgumentException(
+ "responseTimeThreshold millis must be >= 0");
+ }
+ }
+ return responseTimeThresholds;
+ }
+
+ // 10s for level 0, 20s for level 1, 30s for level 2, ...
+ private static long[] getDefaultBackOffResponseTimeThresholds(int numLevels) {
+ long[] ret = new long[numLevels];
+ for (int i = 0; i < ret.length; i++) {
+ ret[i] = 10000*(i+1);
+ }
+ return ret;
+ }
+
+ private static Boolean parseBackOffByResponseTimeEnabled(String ns,
+ Configuration conf) {
+ return conf.getBoolean(ns + "." +
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_KEY,
+ IPC_DECAYSCHEDULER_BACKOFF_RESPONSETIME_ENABLE_DEFAULT);
+ }
+
/**
* Decay the stored counts for each user and clean as necessary.
* This method should be called periodically in order to keep
* counts current.
*/
private void decayCurrentCounts() {
- long total = 0;
- Iterator<Map.Entry<Object, AtomicLong>> it =
- callCounts.entrySet().iterator();
-
- while (it.hasNext()) {
- Map.Entry<Object, AtomicLong> entry = it.next();
- AtomicLong count = entry.getValue();
-
- // Compute the next value by reducing it by the decayFactor
- long currentValue = count.get();
- long nextValue = (long)(currentValue * decayFactor);
- total += nextValue;
- count.set(nextValue);
-
- if (nextValue == 0) {
- // We will clean up unused keys here. An interesting optimization might
- // be to have an upper bound on keyspace in callCounts and only
- // clean once we pass it.
- it.remove();
+ try {
+ long total = 0;
+ Iterator<Map.Entry<Object, AtomicLong>> it =
+ callCounts.entrySet().iterator();
+
+ while (it.hasNext()) {
+ Map.Entry<Object, AtomicLong> entry = it.next();
+ AtomicLong count = entry.getValue();
+
+ // Compute the next value by reducing it by the decayFactor
+ long currentValue = count.get();
+ long nextValue = (long) (currentValue * decayFactor);
+ total += nextValue;
+ count.set(nextValue);
+
+ if (nextValue == 0) {
+ // We will clean up unused keys here. An interesting optimization
+ // might be to have an upper bound on keyspace in callCounts and only
+ // clean once we pass it.
+ it.remove();
+ }
}
- }
- // Update the total so that we remain in sync
- totalCalls.set(total);
+ // Update the total so that we remain in sync
+ totalCalls.set(total);
+
+ // Now refresh the cache of scheduling decisions
+ recomputeScheduleCache();
- // Now refresh the cache of scheduling decisions
- recomputeScheduleCache();
+ // Update average response time with decay
+ updateAverageResponseTime(true);
+ } catch (Exception ex) {
+ LOG.error("decayCurrentCounts exception: " +
+ ExceptionUtils.getFullStackTrace(ex));
+ throw ex;
+ }
}
/**
@@ -324,7 +443,7 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
/**
* Given the number of occurrences, compute a scheduling decision.
* @param occurrences how many occurrences
- * @return scheduling decision from 0 to numQueues - 1
+ * @return scheduling decision from 0 to numLevels - 1
*/
private int computePriorityLevel(long occurrences) {
long totalCallSnapshot = totalCalls.get();
@@ -334,14 +453,14 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
proportion = (double) occurrences / totalCallSnapshot;
}
- // Start with low priority queues, since they will be most common
- for(int i = (numQueues - 1); i > 0; i--) {
+ // Start with low priority levels, since they will be most common
+ for(int i = (numLevels - 1); i > 0; i--) {
if (proportion >= this.thresholds[i - 1]) {
- return i; // We've found our queue number
+ return i; // We've found our level number
}
}
- // If we get this far, we're at queue 0
+ // If we get this far, we're at level 0
return 0;
}
@@ -349,7 +468,7 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
* Returns the priority level for a given identity by first trying the cache,
* then computing it.
* @param identity an object responding to toString and hashCode
- * @return integer scheduling decision from 0 to numQueues - 1
+ * @return integer scheduling decision from 0 to numLevels - 1
*/
private int cachedOrComputedPriorityLevel(Object identity) {
try {
@@ -360,22 +479,29 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
if (scheduleCache != null) {
Integer priority = scheduleCache.get(identity);
if (priority != null) {
+ LOG.debug("Cache priority for: {} with priority: {}", identity,
+ priority);
return priority;
}
}
// Cache was no good, compute it
- return computePriorityLevel(occurrences);
+ int priority = computePriorityLevel(occurrences);
+ LOG.debug("compute priority for " + identity + " priority " + priority);
+ return priority;
+
} catch (InterruptedException ie) {
- LOG.warn("Caught InterruptedException, returning low priority queue");
- return numQueues - 1;
+ LOG.warn("Caught InterruptedException, returning low priority level");
+ LOG.debug("Fallback priority for: {} with priority: {}", identity,
+ numLevels - 1);
+ return numLevels - 1;
}
}
/**
* Compute the appropriate priority for a schedulable based on past requests.
* @param obj the schedulable obj to query and remember
- * @return the queue index which we recommend scheduling in
+ * @return the level index which we recommend scheduling in
*/
@Override
public int getPriorityLevel(Schedulable obj) {
@@ -389,6 +515,73 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
return cachedOrComputedPriorityLevel(identity);
}
+ @Override
+ public boolean shouldBackOff(Schedulable obj) {
+ Boolean backOff = false;
+ if (backOffByResponseTimeEnabled) {
+ int priorityLevel = obj.getPriorityLevel();
+ if (LOG.isDebugEnabled()) {
+ double[] responseTimes = getAverageResponseTime();
+ LOG.debug("Current Caller: {} Priority: {} ",
+ obj.getUserGroupInformation().getUserName(),
+ obj.getPriorityLevel());
+ for (int i = 0; i < numLevels; i++) {
+ LOG.debug("Queue: {} responseTime: {} backoffThreshold: {}", i,
+ responseTimes[i], backOffResponseTimeThresholds[i]);
+ }
+ }
+ // High priority rpc over threshold triggers back off of low priority rpc
+ for (int i = 0; i < priorityLevel + 1; i++) {
+ if (responseTimeAvgInLastWindow.get(i) >
+ backOffResponseTimeThresholds[i]) {
+ backOff = true;
+ break;
+ }
+ }
+ }
+ return backOff;
+ }
+
+ @Override
+ public void addResponseTime(String name, int priorityLevel, int queueTime,
+ int processingTime) {
+ responseTimeCountInCurrWindow.getAndIncrement(priorityLevel);
+ responseTimeTotalInCurrWindow.getAndAdd(priorityLevel,
+ queueTime+processingTime);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("addResponseTime for call: {} priority: {} queueTime: {} " +
+ "processingTime: {} ", name, priorityLevel, queueTime,
+ processingTime);
+ }
+ }
+
+ // Update the cached average response time at the end of decay window
+ void updateAverageResponseTime(boolean enableDecay) {
+ for (int i = 0; i < numLevels; i++) {
+ double averageResponseTime = 0;
+ long totalResponseTime = responseTimeTotalInCurrWindow.get(i);
+ long responseTimeCount = responseTimeCountInCurrWindow.get(i);
+ if (responseTimeCount > 0) {
+ averageResponseTime = (double) totalResponseTime / responseTimeCount;
+ }
+ final double lastAvg = responseTimeAvgInLastWindow.get(i);
+ if (enableDecay && lastAvg > 0.0) {
+ final double decayed = decayFactor * lastAvg + averageResponseTime;
+ responseTimeAvgInLastWindow.set(i, decayed);
+ } else {
+ responseTimeAvgInLastWindow.set(i, averageResponseTime);
+ }
+ responseTimeCountInLastWindow.set(i, responseTimeCount);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updateAverageResponseTime queue: {} Average: {} Count: {}",
+ i, averageResponseTime, responseTimeCount);
+ }
+ // Reset for next decay window
+ responseTimeTotalInCurrWindow.set(i, 0);
+ responseTimeCountInCurrWindow.set(i, 0);
+ }
+ }
+
// For testing
@VisibleForTesting
public double getDecayFactor() { return decayFactor; }
@@ -429,16 +622,21 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
// Weakref for delegate, so we don't retain it forever if it can be GC'd
private WeakReference<DecayRpcScheduler> delegate;
+ private double[] averageResponseTimeDefault;
+ private long[] callCountInLastWindowDefault;
- private MetricsProxy(String namespace) {
+ private MetricsProxy(String namespace, int numLevels) {
+ averageResponseTimeDefault = new double[numLevels];
+ callCountInLastWindowDefault = new long[numLevels];
MBeans.register(namespace, "DecayRpcScheduler", this);
}
- public static synchronized MetricsProxy getInstance(String namespace) {
+ public static synchronized MetricsProxy getInstance(String namespace,
+ int numLevels) {
MetricsProxy mp = INSTANCES.get(namespace);
if (mp == null) {
// We must create one
- mp = new MetricsProxy(namespace);
+ mp = new MetricsProxy(namespace, numLevels);
INSTANCES.put(namespace, mp);
}
return mp;
@@ -487,6 +685,25 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
return scheduler.getTotalCallVolume();
}
}
+
+ @Override
+ public double[] getAverageResponseTime() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return averageResponseTimeDefault;
+ } else {
+ return scheduler.getAverageResponseTime();
+ }
+ }
+
+ public long[] getResponseTimeCountInLastWindow() {
+ DecayRpcScheduler scheduler = delegate.get();
+ if (scheduler == null) {
+ return callCountInLastWindowDefault;
+ } else {
+ return scheduler.getResponseTimeCountInLastWindow();
+ }
+ }
}
public int getUniqueIdentityCount() {
@@ -497,6 +714,23 @@ public class DecayRpcScheduler implements RpcScheduler, DecayRpcSchedulerMXBean
return totalCalls.get();
}
+ public long[] getResponseTimeCountInLastWindow() {
+ long[] ret = new long[responseTimeCountInLastWindow.length()];
+ for (int i = 0; i < responseTimeCountInLastWindow.length(); i++) {
+ ret[i] = responseTimeCountInLastWindow.get(i);
+ }
+ return ret;
+ }
+
+ @Override
+ public double[] getAverageResponseTime() {
+ double[] ret = new double[responseTimeAvgInLastWindow.length()];
+ for (int i = 0; i < responseTimeAvgInLastWindow.length(); i++) {
+ ret[i] = responseTimeAvgInLastWindow.get(i);
+ }
+ return ret;
+ }
+
public String getSchedulingDecisionSummary() {
Map<Object, Integer> decisions = scheduleCacheRef.get();
if (decisions == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java
index 3481f19..fab9b93 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcSchedulerMXBean.java
@@ -27,4 +27,6 @@ public interface DecayRpcSchedulerMXBean {
String getCallVolumeSummary();
int getUniqueIdentityCount();
long getTotalCallVolume();
+ double[] getAverageResponseTime();
+ long[] getResponseTimeCountInLastWindow();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java
new file mode 100644
index 0000000..08f74d4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DefaultRpcScheduler.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ipc;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * No op default RPC scheduler.
+ */
+public class DefaultRpcScheduler implements RpcScheduler {
+ @Override
+ public int getPriorityLevel(Schedulable obj) {
+ return 0;
+ }
+
+ @Override
+ public boolean shouldBackOff(Schedulable obj) {
+ return false;
+ }
+
+ @Override
+ public void addResponseTime(String name, int priorityLevel, int queueTime,
+ int processingTime) {
+ }
+
+ public DefaultRpcScheduler(int priorityLevels, String namespace,
+ Configuration conf) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 0b56243..435c454 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -44,8 +44,9 @@ import org.apache.hadoop.metrics2.util.MBeans;
*/
public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
implements BlockingQueue<E> {
- // Configuration Keys
+ @Deprecated
public static final int IPC_CALLQUEUE_PRIORITY_LEVELS_DEFAULT = 4;
+ @Deprecated
public static final String IPC_CALLQUEUE_PRIORITY_LEVELS_KEY =
"faircallqueue.priority-levels";
@@ -66,9 +67,6 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
}
}
- /* Scheduler picks which queue to place in */
- private RpcScheduler scheduler;
-
/* Multiplexer picks which queue to draw from */
private RpcMultiplexer multiplexer;
@@ -83,8 +81,13 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
* Notes: the FairCallQueue has no fixed capacity. Rather, it has a minimum
* capacity of `capacity` and a maximum capacity of `capacity * number_queues`
*/
- public FairCallQueue(int capacity, String ns, Configuration conf) {
- int numQueues = parseNumQueues(ns, conf);
+ public FairCallQueue(int priorityLevels, int capacity, String ns,
+ Configuration conf) {
+ if(priorityLevels < 1) {
+ throw new IllegalArgumentException("Number of Priority Levels must be " +
+ "at least 1");
+ }
+ int numQueues = priorityLevels;
LOG.info("FairCallQueue is in use with " + numQueues + " queues.");
this.queues = new ArrayList<BlockingQueue<E>>(numQueues);
@@ -95,29 +98,13 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
this.overflowedCalls.add(new AtomicLong(0));
}
- this.scheduler = new DecayRpcScheduler(numQueues, ns, conf);
this.multiplexer = new WeightedRoundRobinMultiplexer(numQueues, ns, conf);
-
// Make this the active source of metrics
MetricsProxy mp = MetricsProxy.getInstance(ns);
mp.setDelegate(this);
}
/**
- * Read the number of queues from the configuration.
- * This will affect the FairCallQueue's overall capacity.
- * @throws IllegalArgumentException on invalid queue count
- */
- private static int parseNumQueues(String ns, Configuration conf) {
- int retval = conf.getInt(ns + "." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY,
- IPC_CALLQUEUE_PRIORITY_LEVELS_DEFAULT);
- if(retval < 1) {
- throw new IllegalArgumentException("numQueues must be at least 1");
- }
- return retval;
- }
-
- /**
* Returns the first non-empty queue with equal or lesser priority
* than <i>startIdx</i>. Wraps around, searching a maximum of N
* queues, where N is this.queues.size().
@@ -144,7 +131,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
/**
* Put and offer follow the same pattern:
- * 1. Get a priorityLevel from the scheduler
+ * 1. Get the assigned priorityLevel from the call by scheduler
* 2. Get the nth sub-queue matching this priorityLevel
* 3. delegate the call to this sub-queue.
*
@@ -154,7 +141,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
*/
@Override
public void put(E e) throws InterruptedException {
- int priorityLevel = scheduler.getPriorityLevel(e);
+ int priorityLevel = e.getPriorityLevel();
final int numLevels = this.queues.size();
while (true) {
@@ -185,7 +172,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
@Override
public boolean offer(E e, long timeout, TimeUnit unit)
throws InterruptedException {
- int priorityLevel = scheduler.getPriorityLevel(e);
+ int priorityLevel = e.getPriorityLevel();
BlockingQueue<E> q = this.queues.get(priorityLevel);
boolean ret = q.offer(e, timeout, unit);
@@ -196,7 +183,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
@Override
public boolean offer(E e) {
- int priorityLevel = scheduler.getPriorityLevel(e);
+ int priorityLevel = e.getPriorityLevel();
BlockingQueue<E> q = this.queues.get(priorityLevel);
boolean ret = q.offer(e);
@@ -436,12 +423,6 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
return calls;
}
- // For testing
- @VisibleForTesting
- public void setScheduler(RpcScheduler newScheduler) {
- this.scheduler = newScheduler;
- }
-
@VisibleForTesting
public void setMultiplexer(RpcMultiplexer newMux) {
this.multiplexer = newMux;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index 692d2b6..071e2e8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -654,13 +654,7 @@ public class ProtobufRpcEngine implements RpcEngine {
String detailedMetricsName = (exception == null) ?
methodName :
exception.getClass().getSimpleName();
- server.rpcMetrics.addRpcQueueTime(qTime);
- server.rpcMetrics.addRpcProcessingTime(processingTime);
- server.rpcDetailedMetrics.addProcessingTime(detailedMetricsName,
- processingTime);
- if (server.isLogSlowRPC()) {
- server.logSlowRpcCalls(methodName, processingTime);
- }
+ server.updateMetrics(detailedMetricsName, qTime, processingTime);
}
return new RpcResponseWrapper(result);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
index a155706..6f93b22 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
@@ -19,11 +19,17 @@
package org.apache.hadoop.ipc;
/**
- * Implement this interface to be used for RPC scheduling in the fair call queues.
+ * Implement this interface to be used for RPC scheduling and backoff.
+ *
*/
public interface RpcScheduler {
/**
* Returns priority level greater than zero as a hint for scheduling.
*/
int getPriorityLevel(Schedulable obj);
+
+ boolean shouldBackOff(Schedulable obj);
+
+ void addResponseTime(String name, int priorityLevel, int queueTime,
+ int processingTime);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java
index 38f3518..3b28d85 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Schedulable.java
@@ -18,11 +18,8 @@
package org.apache.hadoop.ipc;
-import java.nio.ByteBuffer;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.io.Writable;
/**
* Interface which allows extracting information necessary to
@@ -31,4 +28,6 @@ import org.apache.hadoop.io.Writable;
@InterfaceAudience.Private
public interface Schedulable {
public UserGroupInformation getUserGroupInformation();
+
+ int getPriorityLevel();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 1d92865..eb28ad5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -396,6 +396,15 @@ public abstract class Server {
return CurCall.get() != null;
}
+ /**
+ * Return the priority level assigned by call queue to an RPC
+ * Returns 0 in case no priority is assigned.
+ */
+ public static int getPriorityLevel() {
+ Call call = CurCall.get();
+ return call != null? call.getPriorityLevel() : 0;
+ }
+
private String bindAddress;
private int port; // port we listen on
private int handlerCount; // number of handler threads
@@ -482,6 +491,18 @@ public abstract class Server {
}
}
+ void updateMetrics(String name, int queueTime, int processingTime) {
+ rpcMetrics.addRpcQueueTime(queueTime);
+ rpcMetrics.addRpcProcessingTime(processingTime);
+ rpcDetailedMetrics.addProcessingTime(name, processingTime);
+ callQueue.addResponseTime(name, getPriorityLevel(), queueTime,
+ processingTime);
+
+ if (isLogSlowRPC()) {
+ logSlowRpcCalls(name, processingTime);
+ }
+ }
+
/**
* A convenience method to bind to a given address and report
* better exceptions if the address is not a valid host.
@@ -578,6 +599,10 @@ public abstract class Server {
return serviceAuthorizationManager;
}
+ private String getQueueClassPrefix() {
+ return CommonConfigurationKeys.IPC_NAMESPACE + "." + port;
+ }
+
static Class<? extends BlockingQueue<Call>> getQueueClass(
String prefix, Configuration conf) {
String name = prefix + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY;
@@ -585,8 +610,29 @@ public abstract class Server {
return CallQueueManager.convertQueueClass(queueClass, Call.class);
}
- private String getQueueClassPrefix() {
- return CommonConfigurationKeys.IPC_CALLQUEUE_NAMESPACE + "." + port;
+ static Class<? extends RpcScheduler> getSchedulerClass(
+ String prefix, Configuration conf) {
+ String schedulerKeyname = prefix + "." + CommonConfigurationKeys
+ .IPC_SCHEDULER_IMPL_KEY;
+ Class<?> schedulerClass = conf.getClass(schedulerKeyname, null);
+ // Patch the configuration for legacy fcq configuration that does not have
+ // a separate scheduler setting
+ if (schedulerClass == null) {
+ String queueKeyName = prefix + "." + CommonConfigurationKeys
+ .IPC_CALLQUEUE_IMPL_KEY;
+ Class<?> queueClass = conf.getClass(queueKeyName, null);
+ if (queueClass != null) {
+ if (queueClass.getCanonicalName().equals(
+ FairCallQueue.class.getCanonicalName())) {
+ conf.setClass(schedulerKeyname, DecayRpcScheduler.class,
+ RpcScheduler.class);
+ }
+ }
+ }
+ schedulerClass = conf.getClass(schedulerKeyname,
+ DefaultRpcScheduler.class);
+
+ return CallQueueManager.convertSchedulerClass(schedulerClass);
}
/*
@@ -595,7 +641,8 @@ public abstract class Server {
public synchronized void refreshCallQueue(Configuration conf) {
// Create the next queue
String prefix = getQueueClassPrefix();
- callQueue.swapQueue(getQueueClass(prefix, conf), maxQueueSize, prefix, conf);
+ callQueue.swapQueue(getSchedulerClass(prefix, conf),
+ getQueueClass(prefix, conf), maxQueueSize, prefix, conf);
}
/**
@@ -623,6 +670,8 @@ public abstract class Server {
private final byte[] clientId;
private final TraceScope traceScope; // the HTrace scope on the server side
private final CallerContext callerContext; // the call context
+ private int priorityLevel;
+ // the priority level assigned by scheduler, 0 by default
private Call(Call call) {
this(call.callId, call.retryCount, call.rpcRequest, call.connection,
@@ -709,7 +758,16 @@ public abstract class Server {
@Override
public UserGroupInformation getUserGroupInformation() {
return connection.user;
- }
+ }
+
+ @Override
+ public int getPriorityLevel() {
+ return this.priorityLevel;
+ }
+
+ public void setPriorityLevel(int priorityLevel) {
+ this.priorityLevel = priorityLevel;
+ }
}
/** Listens on the socket. Creates jobs for the handler threads*/
@@ -2151,6 +2209,9 @@ public abstract class Server {
rpcRequest, this, ProtoUtil.convert(header.getRpcKind()),
header.getClientId().toByteArray(), traceScope, callerContext);
+ // Save the priority level assignment by the scheduler
+ call.setPriorityLevel(callQueue.getPriorityLevel(call));
+
if (callQueue.isClientBackoffEnabled()) {
// if RPC queue is full, we will ask the RPC client to back off by
// throwing RetriableException. Whether RPC client will honor
@@ -2166,9 +2227,10 @@ public abstract class Server {
private void queueRequestOrAskClientToBackOff(Call call)
throws WrappedRpcServerException, InterruptedException {
- // If rpc queue is full, we will ask the client to back off.
- boolean isCallQueued = callQueue.offer(call);
- if (!isCallQueued) {
+ // If rpc scheduler indicates back off based on performance
+ // degradation such as response time or rpc queue is full,
+ // we will ask the client to back off.
+ if (callQueue.shouldBackOff(call) || !callQueue.offer(call)) {
rpcMetrics.incrClientBackoff();
RetriableException retriableException =
new RetriableException("Server is too busy.");
@@ -2513,6 +2575,7 @@ public abstract class Server {
// Setup appropriate callqueue
final String prefix = getQueueClassPrefix();
this.callQueue = new CallQueueManager<Call>(getQueueClass(prefix, conf),
+ getSchedulerClass(prefix, conf),
getClientBackoffEnable(prefix, conf), maxQueueSize, prefix, conf);
this.secretManager = (SecretManager<TokenIdentifier>) secretManager;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
index a1db6be..a9dbb41 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.io.*;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.RPC.RpcInvoker;
-import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -502,13 +501,12 @@ public class WritableRpcEngine implements RpcEngine {
}
}
}
-
- // Invoke the protocol method
- long startTime = Time.now();
- int qTime = (int) (startTime-receivedTime);
- Exception exception = null;
- try {
+ // Invoke the protocol method
+ long startTime = Time.now();
+ int qTime = (int) (startTime-receivedTime);
+ Exception exception = null;
+ try {
Method method =
protocolImpl.protocolClass.getMethod(call.getMethodName(),
call.getParameterClasses());
@@ -539,27 +537,20 @@ public class WritableRpcEngine implements RpcEngine {
exception = ioe;
throw ioe;
} finally {
- int processingTime = (int) (Time.now() - startTime);
- if (LOG.isDebugEnabled()) {
- String msg = "Served: " + call.getMethodName() +
- " queueTime= " + qTime +
- " procesingTime= " + processingTime;
- if (exception != null) {
- msg += " exception= " + exception.getClass().getSimpleName();
- }
- LOG.debug(msg);
- }
- String detailedMetricsName = (exception == null) ?
- call.getMethodName() :
- exception.getClass().getSimpleName();
- server.rpcMetrics.addRpcQueueTime(qTime);
- server.rpcMetrics.addRpcProcessingTime(processingTime);
- server.rpcDetailedMetrics.addProcessingTime(detailedMetricsName,
- processingTime);
- if (server.isLogSlowRPC()) {
- server.logSlowRpcCalls(call.getMethodName(), processingTime);
+ int processingTime = (int) (Time.now() - startTime);
+ if (LOG.isDebugEnabled()) {
+ String msg = "Served: " + call.getMethodName() +
+ " queueTime= " + qTime + " procesingTime= " + processingTime;
+ if (exception != null) {
+ msg += " exception= " + exception.getClass().getSimpleName();
+ }
+ LOG.debug(msg);
}
- }
+ String detailedMetricsName = (exception == null) ?
+ call.getMethodName() :
+ exception.getClass().getSimpleName();
+ server.updateMetrics(detailedMetricsName, qTime, processingTime);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
index 4d659ac..af9ce1b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
@@ -27,17 +27,37 @@ import java.util.HashMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Test;
public class TestCallQueueManager {
private CallQueueManager<FakeCall> manager;
+ private Configuration conf = new Configuration();
- public class FakeCall {
+ public class FakeCall implements Schedulable {
public final int tag; // Can be used for unique identification
-
+ private int priorityLevel;
+ UserGroupInformation fakeUgi = UserGroupInformation.createRemoteUser
+ ("fakeUser");
public FakeCall(int tag) {
this.tag = tag;
}
+
+ @Override
+ public UserGroupInformation getUserGroupInformation() {
+ return fakeUgi;
+ }
+
+ @Override
+ public int getPriorityLevel() {
+ return priorityLevel;
+ }
+
+ public void setPriorityLevel(int level) {
+ this.priorityLevel = level;
+ }
}
/**
@@ -62,7 +82,9 @@ public class TestCallQueueManager {
try {
// Fill up to max (which is infinite if maxCalls < 0)
while (isRunning && (callsAdded < maxCalls || maxCalls < 0)) {
- cq.put(new FakeCall(this.tag));
+ FakeCall call = new FakeCall(this.tag);
+ call.setPriorityLevel(cq.getPriorityLevel(call));
+ cq.put(call);
callsAdded++;
}
} catch (InterruptedException e) {
@@ -135,7 +157,7 @@ public class TestCallQueueManager {
t.start();
t.join(100);
- assertEquals(putter.callsAdded, numberOfPuts);
+ assertEquals(numberOfPuts, putter.callsAdded);
t.interrupt();
}
@@ -143,23 +165,90 @@ public class TestCallQueueManager {
private static final Class<? extends BlockingQueue<FakeCall>> queueClass
= CallQueueManager.convertQueueClass(LinkedBlockingQueue.class, FakeCall.class);
+ private static final Class<? extends RpcScheduler> schedulerClass
+ = CallQueueManager.convertSchedulerClass(DefaultRpcScheduler.class);
+
@Test
public void testCallQueueCapacity() throws InterruptedException {
- manager = new CallQueueManager<FakeCall>(queueClass, false, 10, "", null);
+ manager = new CallQueueManager<FakeCall>(queueClass, schedulerClass, false,
+ 10, "", conf);
assertCanPut(manager, 10, 20); // Will stop at 10 due to capacity
}
@Test
public void testEmptyConsume() throws InterruptedException {
- manager = new CallQueueManager<FakeCall>(queueClass, false, 10, "", null);
+ manager = new CallQueueManager<FakeCall>(queueClass, schedulerClass, false,
+ 10, "", conf);
assertCanTake(manager, 0, 1); // Fails since it's empty
}
+ static Class<? extends BlockingQueue<FakeCall>> getQueueClass(
+ String prefix, Configuration conf) {
+ String name = prefix + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY;
+ Class<?> queueClass = conf.getClass(name, LinkedBlockingQueue.class);
+ return CallQueueManager.convertQueueClass(queueClass, FakeCall.class);
+ }
+
+ @Test
+ public void testFcqBackwardCompatibility() throws InterruptedException {
+ // Test BackwardCompatibility to ensure existing FCQ deployment still
+ // work without explicitly specifying DecayRpcScheduler
+ Configuration conf = new Configuration();
+ final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0";
+
+ final String queueClassName = "org.apache.hadoop.ipc.FairCallQueue";
+ conf.setStrings(ns + "." + CommonConfigurationKeys.IPC_CALLQUEUE_IMPL_KEY,
+ queueClassName);
+
+ // Specify only Fair Call Queue without a scheduler
+ // Ensure the DecayScheduler will be added to avoid breaking.
+ Class<? extends RpcScheduler> scheduler = Server.getSchedulerClass(ns,
+ conf);
+ assertTrue(scheduler.getCanonicalName().
+ equals("org.apache.hadoop.ipc.DecayRpcScheduler"));
+
+ Class<? extends BlockingQueue<FakeCall>> queue =
+ (Class<? extends BlockingQueue<FakeCall>>) getQueueClass(ns, conf);
+ assertTrue(queue.getCanonicalName().equals(queueClassName));
+
+ manager = new CallQueueManager<FakeCall>(queue, scheduler, false,
+ 2, "", conf);
+
+ // Default FCQ has 4 levels and the max capacity is 2 x 4
+ assertCanPut(manager, 3, 3);
+ }
+
+ @Test
+ public void testSchedulerWithoutFCQ() throws InterruptedException {
+ Configuration conf = new Configuration();
+ // Test DecayedRpcScheduler without FCQ
+ // Ensure the default LinkedBlockingQueue can work with DecayedRpcScheduler
+ final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0";
+ final String schedulerClassName = "org.apache.hadoop.ipc.DecayRpcScheduler";
+ conf.setStrings(ns + "." + CommonConfigurationKeys.IPC_SCHEDULER_IMPL_KEY,
+ schedulerClassName);
+
+ Class<? extends BlockingQueue<FakeCall>> queue =
+ (Class<? extends BlockingQueue<FakeCall>>) getQueueClass(ns, conf);
+ assertTrue(queue.getCanonicalName().equals("java.util.concurrent." +
+ "LinkedBlockingQueue"));
+
+ manager = new CallQueueManager<FakeCall>(queue,
+ Server.getSchedulerClass(ns, conf), false,
+ 3, "", conf);
+
+ // LinkedBlockingQueue with a capacity of 3 can put 3 calls
+ assertCanPut(manager, 3, 3);
+ // LinkedBlockingQueue with a capacity of 3 can't put 1 more call
+ assertCanPut(manager, 0, 1);
+ }
+
@Test(timeout=60000)
public void testSwapUnderContention() throws InterruptedException {
- manager = new CallQueueManager<FakeCall>(queueClass, false, 5000, "", null);
+ manager = new CallQueueManager<FakeCall>(queueClass, schedulerClass, false,
+ 5000, "", conf);
ArrayList<Putter> producers = new ArrayList<Putter>();
ArrayList<Taker> consumers = new ArrayList<Taker>();
@@ -188,7 +277,7 @@ public class TestCallQueueManager {
Thread.sleep(500);
for (int i=0; i < 5; i++) {
- manager.swapQueue(queueClass, 5000, "", null);
+ manager.swapQueue(schedulerClass, queueClass, 5000, "", conf);
}
// Stop the producers
@@ -223,24 +312,50 @@ public class TestCallQueueManager {
}
public static class ExceptionFakeCall {
-
public ExceptionFakeCall() {
- throw new IllegalArgumentException("Exception caused by constructor.!!");
+ throw new IllegalArgumentException("Exception caused by call queue " +
+ "constructor.!!");
+ }
+ }
+
+ public static class ExceptionFakeScheduler {
+ public ExceptionFakeScheduler() {
+ throw new IllegalArgumentException("Exception caused by " +
+ "scheduler constructor.!!");
}
}
- private static final Class<? extends BlockingQueue<ExceptionFakeCall>> exceptionQueueClass = CallQueueManager
- .convertQueueClass(ExceptionFakeCall.class, ExceptionFakeCall.class);
+ private static final Class<? extends RpcScheduler>
+ exceptionSchedulerClass = CallQueueManager.convertSchedulerClass(
+ ExceptionFakeScheduler.class);
+
+ private static final Class<? extends BlockingQueue<ExceptionFakeCall>>
+ exceptionQueueClass = CallQueueManager.convertQueueClass(
+ ExceptionFakeCall.class, ExceptionFakeCall.class);
+
+ @Test
+ public void testCallQueueConstructorException() throws InterruptedException {
+ try {
+ new CallQueueManager<ExceptionFakeCall>(exceptionQueueClass,
+ schedulerClass, false, 10, "", new Configuration());
+ fail();
+ } catch (RuntimeException re) {
+ assertTrue(re.getCause() instanceof IllegalArgumentException);
+ assertEquals("Exception caused by call queue constructor.!!", re
+ .getCause()
+ .getMessage());
+ }
+ }
@Test
- public void testInvocationException() throws InterruptedException {
+ public void testSchedulerConstructorException() throws InterruptedException {
try {
- new CallQueueManager<ExceptionFakeCall>(exceptionQueueClass, false, 10,
- "", null);
+ new CallQueueManager<FakeCall>(queueClass, exceptionSchedulerClass,
+ false, 10, "", new Configuration());
fail();
} catch (RuntimeException re) {
assertTrue(re.getCause() instanceof IllegalArgumentException);
- assertEquals("Exception caused by constructor.!!", re.getCause()
+ assertEquals("Exception caused by scheduler constructor.!!", re.getCause()
.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
index edc3b00..0b0408c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
@@ -18,12 +18,11 @@
package org.apache.hadoop.ipc;
+import static java.lang.Thread.sleep;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.Arrays;
import org.junit.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -31,8 +30,6 @@ import static org.mockito.Mockito.when;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-
public class TestDecayRpcScheduler {
private Schedulable mockCall(String id) {
Schedulable mockCall = mock(Schedulable.class);
@@ -57,30 +54,32 @@ public class TestDecayRpcScheduler {
}
@Test
+ @SuppressWarnings("deprecation")
public void testParsePeriod() {
// By default
scheduler = new DecayRpcScheduler(1, "", new Configuration());
- assertEquals(DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_DEFAULT,
+ assertEquals(DecayRpcScheduler.IPC_SCHEDULER_DECAYSCHEDULER_PERIOD_DEFAULT,
scheduler.getDecayPeriodMillis());
// Custom
Configuration conf = new Configuration();
- conf.setLong("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY,
+ conf.setLong("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY,
1058);
scheduler = new DecayRpcScheduler(1, "ns", conf);
assertEquals(1058L, scheduler.getDecayPeriodMillis());
}
@Test
+ @SuppressWarnings("deprecation")
public void testParseFactor() {
// Default
scheduler = new DecayRpcScheduler(1, "", new Configuration());
- assertEquals(DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT,
+ assertEquals(DecayRpcScheduler.IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT,
scheduler.getDecayFactor(), 0.00001);
// Custom
Configuration conf = new Configuration();
- conf.set("prefix." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
+ conf.set("prefix." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY,
"0.125");
scheduler = new DecayRpcScheduler(1, "prefix", conf);
assertEquals(0.125, scheduler.getDecayFactor(), 0.00001);
@@ -94,6 +93,7 @@ public class TestDecayRpcScheduler {
}
@Test
+ @SuppressWarnings("deprecation")
public void testParseThresholds() {
// Defaults vary by number of queues
Configuration conf = new Configuration();
@@ -111,16 +111,17 @@ public class TestDecayRpcScheduler {
// Custom
conf = new Configuration();
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY,
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY,
"1, 10, 20, 50, 85");
scheduler = new DecayRpcScheduler(6, "ns", conf);
assertEqualDecimalArrays(new double[]{0.01, 0.1, 0.2, 0.5, 0.85}, scheduler.getThresholds());
}
@Test
+ @SuppressWarnings("deprecation")
public void testAccumulate() {
Configuration conf = new Configuration();
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush
scheduler = new DecayRpcScheduler(1, "ns", conf);
assertEquals(0, scheduler.getCallCountSnapshot().size()); // empty first
@@ -138,10 +139,11 @@ public class TestDecayRpcScheduler {
}
@Test
- public void testDecay() {
+ @SuppressWarnings("deprecation")
+ public void testDecay() throws Exception {
Configuration conf = new Configuration();
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "999999999"); // Never
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, "0.5");
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "999999999"); // Never
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, "0.5");
scheduler = new DecayRpcScheduler(1, "ns", conf);
assertEquals(0, scheduler.getTotalCallSnapshot());
@@ -150,6 +152,8 @@ public class TestDecayRpcScheduler {
scheduler.getPriorityLevel(mockCall("A"));
}
+ sleep(1000);
+
for (int i = 0; i < 8; i++) {
scheduler.getPriorityLevel(mockCall("B"));
}
@@ -184,10 +188,11 @@ public class TestDecayRpcScheduler {
}
@Test
+ @SuppressWarnings("deprecation")
public void testPriority() {
Configuration conf = new Configuration();
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_THRESHOLDS_KEY,
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "99999999"); // Never flush
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_THRESHOLDS_KEY,
"25, 50, 75");
scheduler = new DecayRpcScheduler(4, "ns", conf);
@@ -204,10 +209,11 @@ public class TestDecayRpcScheduler {
}
@Test(timeout=2000)
+ @SuppressWarnings("deprecation")
public void testPeriodic() throws InterruptedException {
Configuration conf = new Configuration();
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_PERIOD_KEY, "10");
- conf.set("ns." + DecayRpcScheduler.IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY, "0.5");
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_PERIOD_KEY, "10");
+ conf.set("ns." + DecayRpcScheduler.IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, "0.5");
scheduler = new DecayRpcScheduler(1, "ns", conf);
assertEquals(10, scheduler.getDecayPeriodMillis());
@@ -219,7 +225,7 @@ public class TestDecayRpcScheduler {
// It should eventually decay to zero
while (scheduler.getTotalCallSnapshot() > 0) {
- Thread.sleep(10);
+ sleep(10);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index 2694ba3..4a8ad3b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -37,21 +37,24 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.conf.Configuration;
import org.mockito.Matchers;
-import static org.apache.hadoop.ipc.FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY;
-
public class TestFairCallQueue extends TestCase {
private FairCallQueue<Schedulable> fcq;
- private Schedulable mockCall(String id) {
+ private Schedulable mockCall(String id, int priority) {
Schedulable mockCall = mock(Schedulable.class);
UserGroupInformation ugi = mock(UserGroupInformation.class);
when(ugi.getUserName()).thenReturn(id);
when(mockCall.getUserGroupInformation()).thenReturn(ugi);
+ when(mockCall.getPriorityLevel()).thenReturn(priority);
return mockCall;
}
+ private Schedulable mockCall(String id) {
+ return mockCall(id, 0);
+ }
+
// A scheduler which always schedules into priority zero
private RpcScheduler alwaysZeroScheduler;
{
@@ -60,11 +63,12 @@ public class TestFairCallQueue extends TestCase {
alwaysZeroScheduler = sched;
}
+ @SuppressWarnings("deprecation")
public void setUp() {
Configuration conf = new Configuration();
- conf.setInt("ns." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2);
+ conf.setInt("ns." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2);
- fcq = new FairCallQueue<Schedulable>(5, "ns", conf);
+ fcq = new FairCallQueue<Schedulable>(2, 5, "ns", conf);
}
//
@@ -85,7 +89,6 @@ public class TestFairCallQueue extends TestCase {
}
public void testOfferSucceeds() {
- fcq.setScheduler(alwaysZeroScheduler);
for (int i = 0; i < 5; i++) {
// We can fit 10 calls
@@ -96,7 +99,6 @@ public class TestFairCallQueue extends TestCase {
}
public void testOfferFailsWhenFull() {
- fcq.setScheduler(alwaysZeroScheduler);
for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c"))); }
assertFalse(fcq.offer(mockCall("c"))); // It's full
@@ -107,11 +109,10 @@ public class TestFairCallQueue extends TestCase {
public void testOfferSucceedsWhenScheduledLowPriority() {
// Scheduler will schedule into queue 0 x 5, then queue 1
RpcScheduler sched = mock(RpcScheduler.class);
- when(sched.getPriorityLevel(Matchers.<Schedulable>any())).thenReturn(0, 0, 0, 0, 0, 1, 0);
- fcq.setScheduler(sched);
- for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c"))); }
+ int mockedPriorities[] = {0, 0, 0, 0, 0, 1, 0};
+ for (int i = 0; i < 5; i++) { assertTrue(fcq.offer(mockCall("c", mockedPriorities[i]))); }
- assertTrue(fcq.offer(mockCall("c")));
+ assertTrue(fcq.offer(mockCall("c", mockedPriorities[5])));
assertEquals(6, fcq.size());
}
@@ -121,7 +122,7 @@ public class TestFairCallQueue extends TestCase {
}
public void testPeekNonDestructive() {
- Schedulable call = mockCall("c");
+ Schedulable call = mockCall("c", 0);
assertTrue(fcq.offer(call));
assertEquals(call, fcq.peek());
@@ -130,8 +131,8 @@ public class TestFairCallQueue extends TestCase {
}
public void testPeekPointsAtHead() {
- Schedulable call = mockCall("c");
- Schedulable next = mockCall("b");
+ Schedulable call = mockCall("c", 0);
+ Schedulable next = mockCall("b", 0);
fcq.offer(call);
fcq.offer(next);
@@ -139,15 +140,11 @@ public class TestFairCallQueue extends TestCase {
}
public void testPollTimeout() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
-
assertNull(fcq.poll(10, TimeUnit.MILLISECONDS));
}
public void testPollSuccess() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
-
- Schedulable call = mockCall("c");
+ Schedulable call = mockCall("c", 0);
assertTrue(fcq.offer(call));
assertEquals(call, fcq.poll(10, TimeUnit.MILLISECONDS));
@@ -156,7 +153,6 @@ public class TestFairCallQueue extends TestCase {
}
public void testOfferTimeout() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
for (int i = 0; i < 5; i++) {
assertTrue(fcq.offer(mockCall("c"), 10, TimeUnit.MILLISECONDS));
}
@@ -166,13 +162,11 @@ public class TestFairCallQueue extends TestCase {
assertEquals(5, fcq.size());
}
+ @SuppressWarnings("deprecation")
public void testDrainTo() {
Configuration conf = new Configuration();
- conf.setInt("ns." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2);
- FairCallQueue<Schedulable> fcq2 = new FairCallQueue<Schedulable>(10, "ns", conf);
-
- fcq.setScheduler(alwaysZeroScheduler);
- fcq2.setScheduler(alwaysZeroScheduler);
+ conf.setInt("ns." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2);
+ FairCallQueue<Schedulable> fcq2 = new FairCallQueue<Schedulable>(2, 10, "ns", conf);
// Start with 3 in fcq, to be drained
for (int i = 0; i < 3; i++) {
@@ -185,13 +179,11 @@ public class TestFairCallQueue extends TestCase {
assertEquals(3, fcq2.size());
}
+ @SuppressWarnings("deprecation")
public void testDrainToWithLimit() {
Configuration conf = new Configuration();
- conf.setInt("ns." + IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2);
- FairCallQueue<Schedulable> fcq2 = new FairCallQueue<Schedulable>(10, "ns", conf);
-
- fcq.setScheduler(alwaysZeroScheduler);
- fcq2.setScheduler(alwaysZeroScheduler);
+ conf.setInt("ns." + FairCallQueue.IPC_CALLQUEUE_PRIORITY_LEVELS_KEY, 2);
+ FairCallQueue<Schedulable> fcq2 = new FairCallQueue<Schedulable>(2, 10, "ns", conf);
// Start with 3 in fcq, to be drained
for (int i = 0; i < 3; i++) {
@@ -209,27 +201,23 @@ public class TestFairCallQueue extends TestCase {
}
public void testFirstQueueFullRemainingCapacity() {
- fcq.setScheduler(alwaysZeroScheduler);
while (fcq.offer(mockCall("c"))) ; // Queue 0 will fill up first, then queue 1
assertEquals(5, fcq.remainingCapacity());
}
public void testAllQueuesFullRemainingCapacity() {
- RpcScheduler sched = mock(RpcScheduler.class);
- when(sched.getPriorityLevel(Matchers.<Schedulable>any())).thenReturn(0, 0, 0, 0, 0, 1, 1, 1, 1, 1);
- fcq.setScheduler(sched);
- while (fcq.offer(mockCall("c"))) ;
+ int[] mockedPriorities = {0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0};
+ int i = 0;
+ while (fcq.offer(mockCall("c", mockedPriorities[i++]))) ;
assertEquals(0, fcq.remainingCapacity());
assertEquals(10, fcq.size());
}
public void testQueuesPartialFilledRemainingCapacity() {
- RpcScheduler sched = mock(RpcScheduler.class);
- when(sched.getPriorityLevel(Matchers.<Schedulable>any())).thenReturn(0, 1, 0, 1, 0);
- fcq.setScheduler(sched);
- for (int i = 0; i < 5; i++) { fcq.offer(mockCall("c")); }
+ int[] mockedPriorities = {0, 1, 0, 1, 0};
+ for (int i = 0; i < 5; i++) { fcq.offer(mockCall("c", mockedPriorities[i])); }
assertEquals(5, fcq.remainingCapacity());
assertEquals(5, fcq.size());
@@ -351,16 +339,12 @@ public class TestFairCallQueue extends TestCase {
// Make sure put will overflow into lower queues when the top is full
public void testPutOverflows() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
-
// We can fit more than 5, even though the scheduler suggests the top queue
assertCanPut(fcq, 8, 8);
assertEquals(8, fcq.size());
}
public void testPutBlocksWhenAllFull() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
-
assertCanPut(fcq, 10, 10); // Fill up
assertEquals(10, fcq.size());
@@ -369,12 +353,10 @@ public class TestFairCallQueue extends TestCase {
}
public void testTakeBlocksWhenEmpty() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
assertCanTake(fcq, 0, 1);
}
public void testTakeRemovesCall() throws InterruptedException {
- fcq.setScheduler(alwaysZeroScheduler);
Schedulable call = mockCall("c");
fcq.offer(call);
@@ -383,17 +365,14 @@ public class TestFairCallQueue extends TestCase {
}
public void testTakeTriesNextQueue() throws InterruptedException {
- // Make a FCQ filled with calls in q 1 but empty in q 0
- RpcScheduler q1Scheduler = mock(RpcScheduler.class);
- when(q1Scheduler.getPriorityLevel(Matchers.<Schedulable>any())).thenReturn(1);
- fcq.setScheduler(q1Scheduler);
// A mux which only draws from q 0
RpcMultiplexer q0mux = mock(RpcMultiplexer.class);
when(q0mux.getAndAdvanceCurrentIndex()).thenReturn(0);
fcq.setMultiplexer(q0mux);
- Schedulable call = mockCall("c");
+ // Make a FCQ filled with calls in q 1 but empty in q 0
+ Schedulable call = mockCall("c", 1);
fcq.put(call);
// Take from q1 even though mux said q0, since q0 empty
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d95c6eb3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java
index 1fa0fff..2638412 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIdentityProviders.java
@@ -19,25 +19,15 @@
package org.apache.hadoop.ipc;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Test;
-import org.junit.Before;
-import org.junit.After;
import java.util.List;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.conf.Configuration;
@@ -55,16 +45,20 @@ public class TestIdentityProviders {
}
}
+ @Override
+ public int getPriorityLevel() {
+ return 0;
+ }
}
@Test
public void testPluggableIdentityProvider() {
Configuration conf = new Configuration();
- conf.set(CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY,
+ conf.set(CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY,
"org.apache.hadoop.ipc.UserIdentityProvider");
List<IdentityProvider> providers = conf.getInstances(
- CommonConfigurationKeys.IPC_CALLQUEUE_IDENTITY_PROVIDER_KEY,
+ CommonConfigurationKeys.IPC_IDENTITY_PROVIDER_KEY,
IdentityProvider.class);
assertTrue(providers.size() == 1);
[33/50] [abbrv] hadoop git commit: HADOOP-12169 ListStatus on empty
dir in S3A lists itself instead of returning an empty list. author: Pieter
Reuse. - omitted new S3A subclass
Posted by ae...@apache.org.
HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse. - omitted new S3A subclass
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89c93475
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89c93475
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89c93475
Branch: refs/heads/HDFS-1312
Commit: 89c93475116ee475645cf81cc80f155f830e61de
Parents: 5092c94
Author: Steve Loughran <st...@apache.org>
Authored: Mon Apr 4 17:00:35 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Apr 4 17:02:04 2016 +0100
----------------------------------------------------------------------
.../s3a/TestS3AContractGetFileStatus.java | 31 ++++++++++++++++++++
1 file changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89c93475/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
new file mode 100644
index 0000000..d7b8fe3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+
+public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+}
[48/50] [abbrv] hadoop git commit: YARN-4915. Fix typo in YARN Secure
Containers documentation (Takashi Ohnishi via iwasakims)
Posted by ae...@apache.org.
YARN-4915. Fix typo in YARN Secure Containers documentation (Takashi Ohnishi via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30206346
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30206346
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30206346
Branch: refs/heads/HDFS-1312
Commit: 30206346cf13fe1b7267f86e7c210b77c86b88c9
Parents: 85ec557
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Apr 6 03:47:22 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Apr 6 03:47:22 2016 +0900
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/SecureContainer.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30206346/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
index cd4f913..f7706c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
@@ -114,7 +114,7 @@ min.user.id=1000#Prevent other super-users
`yarn.nodemanager.windows-secure-container-executor.impersonate.allowed` should contain the users that are allowed to create containers in the cluster. These users will be allowed to be impersonated by hadoopwinutilsvc.
- `yarn.nodemanager.windows-secure-container-executor.impersonate.denied` should contain users that are explictly forbiden from creating containers. hadoopwinutilsvc will refuse to impersonate these users.
+ `yarn.nodemanager.windows-secure-container-executor.impersonate.denied` should contain users that are explicitly forbidden from creating containers. hadoopwinutilsvc will refuse to impersonate these users.
`yarn.nodemanager.windows-secure-container-executor.local-dirs` should contain the nodemanager local dirs. hadoopwinutilsvc will allow only file operations under these directories. This should contain the same values as `$yarn.nodemanager.local-dirs, $yarn.nodemanager.log-dirs` but note that hadoopwinutilsvc XML configuration processing does not do substitutions so the value must be the final value. All paths must be absolute and no environment variable substitution will be performed. The paths are compared LOCAL\_INVARIANT case insensitive string comparison, the file path validated must start with one of the paths listed in local-dirs configuration. Use comma as path separator:`,`
[44/50] [abbrv] hadoop git commit: YARN-4893. Fix some intermittent
test failures in TestRMAdminService. Contributed by Brahma Reddy Battula.
Posted by ae...@apache.org.
YARN-4893. Fix some intermittent test failures in TestRMAdminService. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6be28bcc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6be28bcc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6be28bcc
Branch: refs/heads/HDFS-1312
Commit: 6be28bcc461292b24589dae17a235b3eaadc07ed
Parents: 1cbcd4a
Author: Junping Du <ju...@apache.org>
Authored: Tue Apr 5 06:57:26 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Tue Apr 5 06:57:54 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/server/resourcemanager/MockRM.java | 7 +++++--
.../yarn/server/resourcemanager/TestRMAdminService.java | 3 ---
.../hadoop/yarn/server/resourcemanager/TestRMRestart.java | 2 --
.../server/resourcemanager/TestResourceTrackerService.java | 6 ------
.../server/resourcemanager/rmapp/TestNodesListManager.java | 5 ++---
5 files changed, 7 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6be28bcc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index d5b64c1..25c558f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -56,12 +56,12 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -603,6 +603,7 @@ public class MockRM extends ResourceManager {
public MockNM registerNode(String nodeIdStr, int memory) throws Exception {
MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService());
nm.registerNode();
+ drainEvents();
return nm;
}
@@ -611,6 +612,7 @@ public class MockRM extends ResourceManager {
MockNM nm =
new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService());
nm.registerNode();
+ drainEvents();
return nm;
}
@@ -620,6 +622,7 @@ public class MockRM extends ResourceManager {
new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(),
YarnVersionInfo.getVersion());
nm.registerNode(runningApplications);
+ drainEvents();
return nm;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6be28bcc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 4513cbb..5c69411 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -27,9 +27,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -49,7 +47,6 @@ import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
import org.apache.hadoop.yarn.api.records.DecommissionType;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6be28bcc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 9606687..3b9952b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -81,8 +81,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6be28bcc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index dd37b67..68ba7ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -176,7 +176,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
MockNM nm2 = rm.registerNode("host2:5678", 10240);
MockNM nm3 = rm.registerNode("localhost:4433", 1024);
- rm.drainEvents();
int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
@@ -208,7 +207,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.getNodesListManager().refreshNodes(conf);
nm3 = rm.registerNode("localhost:4433", 1024);
- rm.drainEvents();
nodeHeartbeat = nm3.nodeHeartbeat(true);
rm.drainEvents();
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -1027,7 +1025,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
// unhealthy node changed back to healthy
nm2 = rm.registerNode("host2:5678", 5120);
- rm.drainEvents();
response = nm2.nodeHeartbeat(true);
response = nm2.nodeHeartbeat(true);
rm.drainEvents();
@@ -1035,7 +1032,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
// reconnect of node with changed capability
nm1 = rm.registerNode("host2:5678", 10240);
- rm.drainEvents();
response = nm1.nodeHeartbeat(true);
rm.drainEvents();
Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
@@ -1045,7 +1041,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
List<ApplicationId> runningApps = new ArrayList<ApplicationId>();
runningApps.add(ApplicationId.newInstance(1, 0));
nm1 = rm.registerNode("host2:5678", 15360, 2, runningApps);
- rm.drainEvents();
response = nm1.nodeHeartbeat(true);
rm.drainEvents();
Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
@@ -1055,7 +1050,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm1 = new MockNM("host1:1234", 5120, rm.getResourceTrackerService());
nm1.setHttpPort(3);
nm1.registerNode();
- rm.drainEvents();
response = nm1.nodeHeartbeat(true);
response = nm1.nodeHeartbeat(true);
rm.drainEvents();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6be28bcc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
index e747df4..507165d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -236,7 +235,7 @@ public class TestNodesListManager {
* Create dispatcher object
*/
private Dispatcher getDispatcher() {
- Dispatcher dispatcher = new AsyncDispatcher() {
+ Dispatcher dispatcher = new DrainDispatcher() {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public EventHandler getEventHandler() {
[20/50] [abbrv] hadoop git commit: HADOOP-12955. Fix bugs in the
initialization of the ISA-L library JNI bindings (Kai Zheng via cmccabe)
Posted by ae...@apache.org.
HADOOP-12955. Fix bugs in the initialization of the ISA-L library JNI bindings (Kai Zheng via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19639785
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19639785
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19639785
Branch: refs/heads/HDFS-1312
Commit: 19639785f5e9c483558ce585287b9dda9d626263
Parents: 12b11e2
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Mar 31 15:09:11 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Mar 31 15:09:11 2016 -0700
----------------------------------------------------------------------
.../hadoop/util/NativeLibraryChecker.java | 8 ++--
.../hadoop/io/erasurecode/erasure_coder.c | 1 +
.../apache/hadoop/io/erasurecode/isal_load.c | 46 +++++++++-----------
.../apache/hadoop/io/erasurecode/isal_load.h | 6 +--
.../io/erasurecode/jni_erasure_code_native.c | 11 +++--
5 files changed, 34 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19639785/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
index 46f0897..e166bec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
@@ -95,12 +95,12 @@ public class NativeLibraryChecker {
snappyLibraryName = SnappyCodec.getLibraryName();
}
- try {
- isalDetail = ErasureCodeNative.getLoadingFailureReason();
+ isalDetail = ErasureCodeNative.getLoadingFailureReason();
+ if (isalDetail != null) {
+ isalLoaded = false;
+ } else {
isalDetail = ErasureCodeNative.getLibraryName();
isalLoaded = true;
- } catch (UnsatisfiedLinkError e) {
- isalLoaded = false;
}
openSslDetail = OpensslCipher.getLoadingFailureReason();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19639785/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
index b3479bb..b2d856b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
@@ -19,6 +19,7 @@
#include "erasure_code.h"
#include "gf_util.h"
#include "erasure_coder.h"
+#include "dump.h"
#include <stdio.h>
#include <stdlib.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19639785/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c
index 55e8efd..26d8e1a 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c
@@ -78,6 +78,12 @@ static const char* load_functions() {
void load_erasurecode_lib(char* err, size_t err_len) {
const char* errMsg;
+ const char* library = NULL;
+#ifdef UNIX
+ Dl_info dl_info;
+#else
+ LPTSTR filename = NULL;
+#endif
err[0] = '\0';
@@ -111,38 +117,28 @@ void load_erasurecode_lib(char* err, size_t err_len) {
if (errMsg != NULL) {
snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg);
}
-}
-int build_support_erasurecode() {
-#ifdef HADOOP_ISAL_LIBRARY
- return 1;
-#else
- return 0;
-#endif
-}
-
-const char* get_library_name() {
#ifdef UNIX
- Dl_info dl_info;
-
- if (isaLoader->ec_encode_data == NULL) {
- return HADOOP_ISAL_LIBRARY;
- }
-
if(dladdr(isaLoader->ec_encode_data, &dl_info)) {
- return dl_info.dli_fname;
+ library = dl_info.dli_fname;
}
#else
- LPTSTR filename = NULL;
-
- if (isaLoader->libec == NULL) {
- return HADOOP_ISAL_LIBRARY;
- }
-
if (GetModuleFileName(isaLoader->libec, filename, 256) > 0) {
- return filename;
+ library = filename;
}
#endif
- return NULL;
+ if (library == NULL) {
+ library = HADOOP_ISAL_LIBRARY;
+ }
+
+ isaLoader->libname = strdup(library);
}
+
+int build_support_erasurecode() {
+#ifdef HADOOP_ISAL_LIBRARY
+ return 1;
+#else
+ return 0;
+#endif
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19639785/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h
index 635706d..7cb7a6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h
@@ -78,6 +78,7 @@ typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned c
typedef struct __IsaLibLoader {
// The loaded library handle
void* libec;
+ char* libname;
__d_gf_mul gf_mul;
__d_gf_inv gf_inv;
@@ -134,11 +135,6 @@ static FARPROC WINAPI myDlsym(HMODULE handle, LPCSTR symbol) {
int build_support_erasurecode();
/**
- * Get the library name possibly of full path.
- */
-const char* get_library_name();
-
-/**
* Initialize and load erasure code library, returning error message if any.
*
* @param err The err message buffer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19639785/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
index eb09e73..b46e8a9 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
@@ -22,6 +22,7 @@
#include "org_apache_hadoop.h"
#include "jni_common.h"
+#include "isal_load.h"
#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h"
#ifdef UNIX
@@ -37,9 +38,11 @@ Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary
JNIEXPORT jstring JNICALL
Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName
(JNIEnv *env, jclass myclass) {
- char* libName = get_library_name();
- if (libName == NULL) {
- libName = "Unavailable";
+ if (isaLoader == NULL) {
+ THROW(env, "java/lang/UnsatisfiedLinkError",
+ "Unavailable: library not loaded yet");
+ return (jstring)NULL;
}
- return (*env)->NewStringUTF(env, libName);
+
+ return (*env)->NewStringUTF(env, isaLoader->libname);
}
[16/50] [abbrv] hadoop git commit: YARN-4183. Clarify the behavior of
timeline service config properties (Naganarasimha G R via sjlee)
Posted by ae...@apache.org.
YARN-4183. Clarify the behavior of timeline service config properties (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d67420d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d67420d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d67420d
Branch: refs/heads/HDFS-1312
Commit: 6d67420dbc5c6097216fa40fcec8ed626b2bae14
Parents: d95c6eb
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Mar 31 10:49:03 2016 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Thu Mar 31 10:49:03 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 13 ++++++++++---
.../src/site/markdown/TimelineServer.md | 6 +++---
2 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d67420d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 33cd919..cb3c73a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1874,8 +1874,12 @@
</property>
<property>
- <description>Indicate to clients whether timeline service is enabled or not.
- If enabled, clients will put entities and events to the timeline server.
+ <description>
+ In the server side it indicates whether timeline service is enabled or not.
+ And in the client side, users can enable it to indicate whether client wants
+ to use timeline service. If it's enabled in the client side along with
+ security, then yarn client tries to fetch the delegation tokens for the
+ timeline server.
</description>
<name>yarn.timeline-service.enabled</name>
<value>false</value>
@@ -2027,7 +2031,10 @@
</property>
<property>
- <description>Client policy for whether timeline operations are non-fatal</description>
+ <description>Client policy for whether timeline operations are non-fatal.
+ Should the failure to obtain a delegation token be considered an application
+ failure (option = false), or should the client attempt to continue to
+ publish information without it (option=true)</description>
<name>yarn.timeline-service.client.best-effort</name>
<value>false</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d67420d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 8ef7d9a..9283e58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -141,7 +141,7 @@ and cluster operators.
| Configuration Property | Description |
|:---- |:---- |
-| `yarn.timeline-service.enabled` | Indicate to clients whether Timeline service is enabled or not. If enabled, the `TimelineClient` library used by applications will post entities and events to the Timeline server. Defaults to `false`. |
+| `yarn.timeline-service.enabled` | In the server side it indicates whether timeline service is enabled or not. And in the client side, users can enable it to indicate whether client wants to use timeline service. If it's enabled in the client side along with security, then yarn client tries to fetch the delegation tokens for the timeline server. Defaults to `false`. |
| `yarn.resourcemanager.system-metrics-publisher.enabled` | The setting that controls whether or not YARN system metrics are published on the timeline server by RM. Defaults to `false`. |
| `yarn.timeline-service.generic-application-history.enabled` | Indicate to clients whether to query generic application data from timeline history-service or not. If not enabled then application data is queried only from Resource Manager. Defaults to `false`. |
@@ -150,7 +150,7 @@ and cluster operators.
| Configuration Property | Description |
|:---- |:---- |
| `yarn.timeline-service.store-class` | Store class name for timeline store. Defaults to `org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore`. |
-| `yarn.timeline-service.leveldb-timeline-store.path` | Store file name for leveldb timeline store. Defaults to `${hadoop.tmp.dir}/yarn/timelin`e. |
+| `yarn.timeline-service.leveldb-timeline-store.path` | Store file name for leveldb timeline store. Defaults to `${hadoop.tmp.dir}/yarn/timeline`. |
| `yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms` | Length of time to wait between deletion cycles of leveldb timeline store in milliseconds. Defaults to `300000`. |
| `yarn.timeline-service.leveldb-timeline-store.read-cache-size` | Size of read cache for uncompressed blocks for leveldb timeline store in bytes. Defaults to `104857600`. |
| `yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size` | Size of cache for recently read entity start times for leveldb timeline store in number of entities. Defaults to `10000`. |
@@ -215,7 +215,7 @@ to `kerberos`, after which the following configuration options are available:
| `yarn.timeline-service.delegation.key.update-interval` | Defaults to `86400000` (1 day). |
| `yarn.timeline-service.delegation.token.renew-interval` | Defaults to `86400000` (1 day). |
| `yarn.timeline-service.delegation.token.max-lifetime` | Defaults to `604800000` (7 days). |
-| `yarn.timeline-service.best-effort` | Should the failure to obtain a delegation token be considered an application failure (option = false), or should the client attempt to continue to publish information without it (option=true). Default: `false` |
+| `yarn.timeline-service.client.best-effort` | Should the failure to obtain a delegation token be considered an application failure (option = false), or should the client attempt to continue to publish information without it (option=true). Default: `false` |
#### Enabling the timeline service and the generic history service