You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2018/02/12 02:38:32 UTC
[1/4] hadoop git commit: YARN-7906. Fix mvn site fails with error:
Multiple sources of package comments found for package
o.a.h.y.client.api.impl. (Akira Ajisaka via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk a08c04883 -> d02e42cee
YARN-7906. Fix mvn site fails with error: Multiple sources of package comments found for package o.a.h.y.client.api.impl. (Akira Ajisaka via wangda)
Change-Id: I20221d97446e97f208d587eacbc60448c11ffd48
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e795833d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e795833d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e795833d
Branch: refs/heads/trunk
Commit: e795833d8c1981cab85a10b4e516cd0c5423c792
Parents: a08c048
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Feb 12 10:25:22 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Feb 12 10:25:22 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/api/resource/package-info.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e795833d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
index 660dc02..a9388b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
@@ -18,6 +18,4 @@
/**
* API related to resources.
*/
-@InterfaceAudience.Private
package org.apache.hadoop.yarn.api.resource;
-import org.apache.hadoop.classification.InterfaceAudience;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/4] hadoop git commit: YARN-5848. Remove unnecessary
public/crossdomain.xml from YARN UIv2 sub project. (Sunil G via wangda)
Posted by wa...@apache.org.
YARN-5848. Remove unnecessary public/crossdomain.xml from YARN UIv2 sub project. (Sunil G via wangda)
Change-Id: Ie295f88232192e6b520c335b0332383cc6a232c0
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/789a185c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/789a185c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/789a185c
Branch: refs/heads/trunk
Commit: 789a185c16351d2343e075413a50eb3e5849cc5f
Parents: e795833
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Feb 12 10:27:15 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Feb 12 10:27:15 2018 +0800
----------------------------------------------------------------------
.../hadoop-yarn-ui/public/crossdomain.xml | 15 ---------------
1 file changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/789a185c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/public/crossdomain.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/public/crossdomain.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/public/crossdomain.xml
deleted file mode 100644
index 0c16a7a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/public/crossdomain.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version="1.0"?>
-<!DOCTYPE cross-domain-policy SYSTEM "http://www.adobe.com/xml/dtds/cross-domain-policy.dtd">
-<cross-domain-policy>
- <!-- Read this: www.adobe.com/devnet/articles/crossdomain_policy_file_spec.html -->
-
- <!-- Most restrictive policy: -->
- <site-control permitted-cross-domain-policies="none"/>
-
- <!-- Least restrictive policy: -->
- <!--
- <site-control permitted-cross-domain-policies="all"/>
- <allow-access-from domain="*" to-ports="*" secure="false"/>
- <allow-http-request-headers-from domain="*" headers="*" secure="false"/>
- -->
-</cross-domain-policy>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/4] hadoop git commit: YARN-7697. NM goes down with OOM due to leak
in log-aggregation. (Xuan Gong via wangda)
Posted by wa...@apache.org.
YARN-7697. NM goes down with OOM due to leak in log-aggregation. (Xuan Gong via wangda)
Change-Id: Ie4fc7979d834e25f37a033c314f3efceeb8f4a9e
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4c98579
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4c98579
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4c98579
Branch: refs/heads/trunk
Commit: d4c98579e36df7eeb788352d7b76cd2c7448c511
Parents: 789a185
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Feb 12 10:28:35 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Feb 12 10:28:35 2018 +0800
----------------------------------------------------------------------
.../LogAggregationFileController.java | 7 +-
.../ifile/IndexedFileAggregatedLogsBlock.java | 2 +-
.../LogAggregationIndexedFileController.java | 69 +++++++++++++-------
.../tfile/LogAggregationTFileController.java | 5 +-
...TestLogAggregationFileControllerFactory.java | 5 +-
.../TestLogAggregationIndexFileController.java | 21 ++++++
6 files changed, 79 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c98579/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 0590535..aeef574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -226,10 +226,12 @@ public abstract class LogAggregationFileController {
* Returns the owner of the application.
*
* @param aggregatedLogPath the aggregatedLog path
+ * @param appId the ApplicationId
* @return the application owner
* @throws IOException if we can not get the application owner
*/
- public abstract String getApplicationOwner(Path aggregatedLogPath)
+ public abstract String getApplicationOwner(Path aggregatedLogPath,
+ ApplicationId appId)
throws IOException;
/**
@@ -237,11 +239,12 @@ public abstract class LogAggregationFileController {
* found.
*
* @param aggregatedLogPath the aggregatedLog path.
+ * @param appId the ApplicationId
* @return a map of the Application ACLs.
* @throws IOException if we can not get the application acls
*/
public abstract Map<ApplicationAccessType, String> getApplicationAcls(
- Path aggregatedLogPath) throws IOException;
+ Path aggregatedLogPath, ApplicationId appId) throws IOException;
/**
* Verify and create the remote log directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c98579/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
index 6d48d7a..c53ffcc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
@@ -135,7 +135,7 @@ public class IndexedFileAggregatedLogsBlock extends LogAggregationHtmlBlock {
IndexedLogsMeta indexedLogsMeta = null;
try {
indexedLogsMeta = fileController.loadIndexedLogsMeta(
- thisNodeFile.getPath(), endIndex);
+ thisNodeFile.getPath(), endIndex, appId);
} catch (Exception ex) {
// DO NOTHING
LOG.warn("Can not load log meta from the log file:"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c98579/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
index 800c0a2..56bae26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
@@ -284,16 +284,8 @@ public class LogAggregationIndexedFileController
currentRemoteLogFile.getName())) {
overwriteCheckSum = false;
long endIndex = checksumFileInputStream.readLong();
- IndexedLogsMeta recoveredLogsMeta = null;
- try {
- truncateFileWithRetries(fc, currentRemoteLogFile,
- endIndex);
- recoveredLogsMeta = loadIndexedLogsMeta(
- currentRemoteLogFile);
- } catch (Exception ex) {
- recoveredLogsMeta = loadIndexedLogsMeta(
- currentRemoteLogFile, endIndex);
- }
+ IndexedLogsMeta recoveredLogsMeta = loadIndexedLogsMeta(
+ currentRemoteLogFile, endIndex, appId);
if (recoveredLogsMeta != null) {
indexedLogsMeta = recoveredLogsMeta;
}
@@ -524,11 +516,11 @@ public class LogAggregationIndexedFileController
IndexedLogsMeta indexedLogsMeta = null;
try {
indexedLogsMeta = loadIndexedLogsMeta(thisNodeFile.getPath(),
- endIndex);
+ endIndex, appId);
} catch (Exception ex) {
// DO NOTHING
LOG.warn("Can not load log meta from the log file:"
- + thisNodeFile.getPath());
+ + thisNodeFile.getPath() + "\n" + ex.getMessage());
continue;
}
if (indexedLogsMeta == null) {
@@ -636,14 +628,14 @@ public class LogAggregationIndexedFileController
endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta current = loadIndexedLogsMeta(
- thisNodeFile.getPath(), endIndex);
+ thisNodeFile.getPath(), endIndex, appId);
if (current != null) {
listOfLogsMeta.add(current);
}
} catch (IOException ex) {
// DO NOTHING
LOG.warn("Can not get log meta from the log file:"
- + thisNodeFile.getPath());
+ + thisNodeFile.getPath() + "\n" + ex.getMessage());
}
}
for (IndexedLogsMeta indexedLogMeta : listOfLogsMeta) {
@@ -721,6 +713,7 @@ public class LogAggregationIndexedFileController
checkSumFiles.put(nodeName, Long.valueOf(index));
}
} catch (IOException ex) {
+ LOG.warn(ex.getMessage());
continue;
} finally {
IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
@@ -773,25 +766,26 @@ public class LogAggregationIndexedFileController
}
@Override
- public String getApplicationOwner(Path aggregatedLogPath)
+ public String getApplicationOwner(Path aggregatedLogPath,
+ ApplicationId appId)
throws IOException {
if (this.cachedIndexedLogsMeta == null
|| !this.cachedIndexedLogsMeta.getRemoteLogPath()
.equals(aggregatedLogPath)) {
this.cachedIndexedLogsMeta = new CachedIndexedLogsMeta(
- loadIndexedLogsMeta(aggregatedLogPath), aggregatedLogPath);
+ loadIndexedLogsMeta(aggregatedLogPath, appId), aggregatedLogPath);
}
return this.cachedIndexedLogsMeta.getCachedIndexedLogsMeta().getUser();
}
@Override
public Map<ApplicationAccessType, String> getApplicationAcls(
- Path aggregatedLogPath) throws IOException {
+ Path aggregatedLogPath, ApplicationId appId) throws IOException {
if (this.cachedIndexedLogsMeta == null
|| !this.cachedIndexedLogsMeta.getRemoteLogPath()
.equals(aggregatedLogPath)) {
this.cachedIndexedLogsMeta = new CachedIndexedLogsMeta(
- loadIndexedLogsMeta(aggregatedLogPath), aggregatedLogPath);
+ loadIndexedLogsMeta(aggregatedLogPath, appId), aggregatedLogPath);
}
return this.cachedIndexedLogsMeta.getCachedIndexedLogsMeta().getAcls();
}
@@ -804,8 +798,8 @@ public class LogAggregationIndexedFileController
}
@Private
- public IndexedLogsMeta loadIndexedLogsMeta(Path remoteLogPath, long end)
- throws IOException {
+ public IndexedLogsMeta loadIndexedLogsMeta(Path remoteLogPath, long end,
+ ApplicationId appId) throws IOException {
FileContext fileContext =
FileContext.getFileContext(remoteLogPath.toUri(), conf);
FSDataInputStream fsDataIStream = null;
@@ -816,8 +810,36 @@ public class LogAggregationIndexedFileController
}
long fileLength = end < 0 ? fileContext.getFileStatus(
remoteLogPath).getLen() : end;
+
fsDataIStream.seek(fileLength - Integer.SIZE/ Byte.SIZE - UUID_LENGTH);
int offset = fsDataIStream.readInt();
+ // If the offset/log meta size is larger than 64M,
+ // output a warn message for better debug.
+ if (offset > 64 * 1024 * 1024) {
+ LOG.warn("The log meta size read from " + remoteLogPath
+ + " is " + offset);
+ }
+
+ // Load UUID and make sure the UUID is correct.
+ byte[] uuidRead = new byte[UUID_LENGTH];
+ int uuidReadLen = fsDataIStream.read(uuidRead);
+ if (this.uuid == null) {
+ this.uuid = createUUID(appId);
+ }
+ if (uuidReadLen != UUID_LENGTH || !Arrays.equals(this.uuid, uuidRead)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("the length of loaded UUID:" + uuidReadLen);
+ LOG.debug("the loaded UUID:" + new String(uuidRead,
+ Charset.forName("UTF-8")));
+ LOG.debug("the expected UUID:" + new String(this.uuid,
+ Charset.forName("UTF-8")));
+ }
+ throw new IOException("The UUID from "
+ + remoteLogPath + " is not correct. The offset of loaded UUID is "
+ + (fileLength - UUID_LENGTH));
+ }
+
+ // Load Log Meta
byte[] array = new byte[offset];
fsDataIStream.seek(
fileLength - offset - Integer.SIZE/ Byte.SIZE - UUID_LENGTH);
@@ -833,9 +855,9 @@ public class LogAggregationIndexedFileController
}
}
- private IndexedLogsMeta loadIndexedLogsMeta(Path remoteLogPath)
- throws IOException {
- return loadIndexedLogsMeta(remoteLogPath, -1);
+ private IndexedLogsMeta loadIndexedLogsMeta(Path remoteLogPath,
+ ApplicationId appId) throws IOException {
+ return loadIndexedLogsMeta(remoteLogPath, -1, appId);
}
/**
@@ -1040,6 +1062,7 @@ public class LogAggregationIndexedFileController
this.out = compressAlgo.createCompressionStream(
fsBufferedOutput, compressor, 0);
} catch (IOException e) {
+ LOG.warn(e.getMessage());
compressAlgo.returnCompressor(compressor);
throw e;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c98579/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
index 5064e26..a4f50d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
@@ -335,14 +335,15 @@ public class LogAggregationTFileController
}
@Override
- public String getApplicationOwner(Path aggregatedLog) throws IOException {
+ public String getApplicationOwner(Path aggregatedLog, ApplicationId appId)
+ throws IOException {
createTFileLogReader(aggregatedLog);
return this.tfReader.getLogReader().getApplicationOwner();
}
@Override
public Map<ApplicationAccessType, String> getApplicationAcls(
- Path aggregatedLog) throws IOException {
+ Path aggregatedLog, ApplicationId appId) throws IOException {
createTFileLogReader(aggregatedLog);
return this.tfReader.getLogReader().getApplicationAcls();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c98579/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
index 2d0864a..99aca1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
@@ -194,14 +194,15 @@ public class TestLogAggregationFileControllerFactory {
}
@Override
- public String getApplicationOwner(Path aggregatedLogPath)
+ public String getApplicationOwner(Path aggregatedLogPath,
+ ApplicationId appId)
throws IOException {
return null;
}
@Override
public Map<ApplicationAccessType, String> getApplicationAcls(
- Path aggregatedLogPath) throws IOException {
+ Path aggregatedLogPath, ApplicationId appId) throws IOException {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4c98579/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
index 7d0205b..9c02c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
@@ -55,7 +55,9 @@ import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.junit.After;
@@ -219,6 +221,25 @@ public class TestLogAggregationIndexFileController {
}
sysOutStream.reset();
+ Configuration factoryConf = new Configuration(conf);
+ factoryConf.set("yarn.log-aggregation.file-formats", "Indexed");
+ factoryConf.set("yarn.log-aggregation.file-controller.Indexed.class",
+ "org.apache.hadoop.yarn.logaggregation.filecontroller.ifile"
+ + ".LogAggregationIndexedFileController");
+ LogAggregationFileControllerFactory factory =
+ new LogAggregationFileControllerFactory(factoryConf);
+ LogAggregationFileController fileController = factory
+ .getFileControllerForRead(appId, USER_UGI.getShortUserName());
+ Assert.assertTrue(fileController instanceof
+ LogAggregationIndexedFileController);
+ foundLogs = fileController.readAggregatedLogs(logRequest, System.out);
+ Assert.assertTrue(foundLogs);
+ for (String logType : logTypes) {
+ Assert.assertTrue(sysOutStream.toString().contains(logMessage(
+ containerId, logType)));
+ }
+ sysOutStream.reset();
+
// create a checksum file
Path checksumFile = new Path(fileFormat.getRemoteAppLogDir(
appId, USER_UGI.getShortUserName()),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/4] hadoop git commit: YARN-7739. DefaultAMSProcessor should
properly check customized resource types against minimum/maximum allocation.
(wangda)
Posted by wa...@apache.org.
YARN-7739. DefaultAMSProcessor should properly check customized resource types against minimum/maximum allocation. (wangda)
Change-Id: I10cc9341237d9a2fc0f8c855efb98a36b91389e2
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d02e42ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d02e42ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d02e42ce
Branch: refs/heads/trunk
Commit: d02e42cee4a08a47ed2835f7a4a100daaa95833f
Parents: d4c9857
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Feb 12 10:29:37 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Feb 12 10:29:37 2018 +0800
----------------------------------------------------------------------
.../scheduler/SchedulerUtils.java | 36 ++--
.../TestApplicationMasterService.java | 190 +++++++++++++++++++
.../scheduler/capacity/TestUtils.java | 8 +-
3 files changed, 214 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d02e42ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 32f5824..0080a29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -276,23 +278,23 @@ public class SchedulerUtils {
throw new InvalidResourceRequestException(ye);
}
- if (resReq.getCapability().getMemorySize() < 0 ||
- resReq.getCapability().getMemorySize() > maximumResource.getMemorySize()) {
- throw new InvalidResourceRequestException("Invalid resource request"
- + ", requested memory < 0"
- + ", or requested memory > max configured"
- + ", requestedMemory=" + resReq.getCapability().getMemorySize()
- + ", maxMemory=" + maximumResource.getMemorySize());
- }
- if (resReq.getCapability().getVirtualCores() < 0 ||
- resReq.getCapability().getVirtualCores() >
- maximumResource.getVirtualCores()) {
- throw new InvalidResourceRequestException("Invalid resource request"
- + ", requested virtual cores < 0"
- + ", or requested virtual cores > max configured"
- + ", requestedVirtualCores="
- + resReq.getCapability().getVirtualCores()
- + ", maxVirtualCores=" + maximumResource.getVirtualCores());
+ Resource requestedResource = resReq.getCapability();
+ for (int i = 0; i < ResourceUtils.getNumberOfKnownResourceTypes(); i++) {
+ ResourceInformation reqRI = requestedResource.getResourceInformation(i);
+ ResourceInformation maxRI = maximumResource.getResourceInformation(i);
+ if (reqRI.getValue() < 0 || reqRI.getValue() > maxRI.getValue()) {
+ throw new InvalidResourceRequestException(
+ "Invalid resource request, requested resource type=[" + reqRI
+ .getName()
+ + "] < 0 or greater than maximum allowed allocation. Requested "
+ + "resource=" + requestedResource
+ + ", maximum allowed allocation=" + maximumResource
+ + ", please note that maximum allowed allocation is calculated "
+ + "by scheduler based on maximum resource of registered "
+ + "NodeManagers, which might be less than configured "
+ + "maximum allocation=" + ResourceUtils
+ .getResourceTypesMaximumAllocation());
+ }
}
String labelExp = resReq.getNodeLabelExpression();
// we don't allow specify label expression other than resourceName=ANY now
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d02e42ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 421ddbc..ceaf236 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static java.lang.Thread.sleep;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
@@ -41,6 +44,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRespons
import org.apache.hadoop.yarn.api.protocolrecords
.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -48,27 +52,39 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.MockResourceProfileManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
@@ -665,6 +681,180 @@ public class TestApplicationMasterService {
rm.stop();
}
+ @Test(timeout = 300000)
+ public void testCSValidateRequestCapacityAgainstMinMaxAllocation()
+ throws Exception {
+ testValidateRequestCapacityAgainstMinMaxAllocation(CapacityScheduler.class);
+ }
+
+ @Test(timeout = 300000)
+ public void testFSValidateRequestCapacityAgainstMinMaxAllocation()
+ throws Exception {
+ testValidateRequestCapacityAgainstMinMaxAllocation(FairScheduler.class);
+ }
+
+ private void testValidateRequestCapacityAgainstMinMaxAllocation(Class<?> schedulerCls)
+ throws Exception {
+
+ // Initialize resource map for 2 types.
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setResourceComparator(DominantResourceCalculator.class);
+
+ YarnConfiguration conf = new YarnConfiguration(csconf);
+ // Don't reset resource types since we have already configured resource
+ // types
+ conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, schedulerCls,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
+
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils
+ .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null));
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ // Now request resource, memory > allowed
+ boolean exception = false;
+ try {
+ am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
+ Resource.newInstance(9 * GB, 1)).numContainers(1).resourceName("*")
+ .build()), null);
+ } catch (InvalidResourceRequestException e) {
+ exception = true;
+ }
+ Assert.assertTrue(exception);
+
+ exception = false;
+ try {
+ // Now request resource, vcore > allowed
+ am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
+ Resource.newInstance(8 * GB, 18)).numContainers(1).resourceName("*")
+ .build()), null);
+ } catch (InvalidResourceRequestException e) {
+ exception = true;
+ }
+ Assert.assertTrue(exception);
+
+ rm.close();
+ }
+
+ @Test(timeout = 300000)
+ public void testValidateRequestCapacityAgainstMinMaxAllocationFor3rdResourceTypes()
+ throws Exception {
+
+ // Initialize resource map for 2 types.
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ ResourceInformation res_1 = ResourceInformation.newInstance("res_1",
+ ResourceInformation.VCORES.getUnits(), 0, 4);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put("res_1", res_1);
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setResourceComparator(DominantResourceCalculator.class);
+
+ YarnConfiguration conf = new YarnConfiguration(csconf);
+ // Don't reset resource types since we have already configured resource
+ // types
+ conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
+
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("default");
+
+ MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils
+ .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ ImmutableMap.of("res_1", 4)));
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ Assert.assertEquals(Resource.newInstance(1 * GB, 1),
+ leafQueue.getUsedResources());
+
+ // Now request resource, memory > allowed
+ boolean exception = false;
+ try {
+ am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
+ TestUtils.createResource(9 * GB, 1, ImmutableMap.of("res_1", 1)))
+ .numContainers(1).resourceName("*").build()), null);
+ } catch (InvalidResourceRequestException e) {
+ exception = true;
+ }
+ Assert.assertTrue(exception);
+
+ exception = false;
+ try {
+ // Now request resource, vcore > allowed
+ am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
+ TestUtils.createResource(8 * GB, 18, ImmutableMap.of("res_1", 1)))
+ .numContainers(1).resourceName("*")
+ .build()), null);
+ } catch (InvalidResourceRequestException e) {
+ exception = true;
+ }
+ Assert.assertTrue(exception);
+
+ exception = false;
+ try {
+ // Now request resource, res_1 > allowed
+ am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
+ TestUtils.createResource(8 * GB, 1, ImmutableMap.of("res_1", 100)))
+ .numContainers(1).resourceName("*")
+ .build()), null);
+ } catch (InvalidResourceRequestException e) {
+ exception = true;
+ }
+ Assert.assertTrue(exception);
+
+ rm.close();
+ }
+
private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
RMContainer rmContainer = cs.getRMContainer(containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d02e42ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 542ba3e..7180e24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -478,9 +478,11 @@ public class TestUtils {
public static Resource createResource(long memory, int vcores,
Map<String, Integer> nameToValues) {
Resource res = Resource.newInstance(memory, vcores);
- for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) {
- res.setResourceInformation(entry.getKey(), ResourceInformation
- .newInstance(entry.getKey(), "", entry.getValue()));
+ if (nameToValues != null) {
+ for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) {
+ res.setResourceInformation(entry.getKey(), ResourceInformation
+ .newInstance(entry.getKey(), "", entry.getValue()));
+ }
}
return res;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org